(ns dev
  (:require
   [clojure.core.async :as a]
   [clojure.string :as str]
   [dev.debug-qp :as debug-qp]
   [dev.explain :as dev.explain]
   [dev.model-tracking :as model-tracking]
   [honey.sql :as sql]
   [java-time :as t]
   [malli.dev :as malli-dev]
   [metabase.api.common :as api]
   [metabase.config :as config]
   [metabase.core :as mbc]
   [metabase.db.connection :as mdb.connection]
   [metabase.db.env :as mdb.env]
   [metabase.db.setup :as mdb.setup]
   [metabase.driver :as driver]
   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
   [metabase.models.database :refer [Database]]
   [metabase.query-processor :as qp]
   [metabase.query-processor.timezone :as qp.timezone]
   [metabase.server :as server]
   [metabase.server.handler :as handler]
   [metabase.sync :as sync]
   [metabase.test :as mt]
   [metabase.test.data.impl :as data.impl]
   [metabase.util :as u]
   [metabase.util.log :as log]
   [methodical.core :as methodical]
   [potemkin :as p]
   [toucan2.connection :as t2.connection]
   [toucan2.core :as t2]
   [toucan2.pipeline :as t2.pipeline]))

(set! *warn-on-reflection* true)

(comment
  debug-qp/keep-me
  model-tracking/keep-me)

(defn tap>-spy [x]
  (doto x tap>))

(p/import-vars
 [debug-qp
  process-query-debug
  pprint-sql]
 [dev.explain
  explain-query]
 [model-tracking
  track!
  untrack!
  untrack-all!
  reset-changes!
  changes])

(def initialized?
  (atom nil))

(defn init!
  []
  (mbc/init!)
  (reset! initialized? true))

(defn migration-timestamp
  []
  (t/format (t/formatter "yyyy-MM-dd'T'HH:mm:ss") (t/zoned-date-time (t/zone-id "UTC"))))

(defn deleted-inmem-databases
  []
  (let [h2-dbs (t2/select :model/Database :engine :h2)
        in-memory? (fn [db] (some-> db :details :db (str/starts-with? "mem:")))
        can-connect? (fn [db]
                       (binding [metabase.driver.h2/*allow-testing-h2-connections* true]
                         (try
                           (driver/can-connect? :h2 (:details db))
                           (catch org.h2.jdbc.JdbcSQLNonTransientConnectionException _
                             false)
                           (catch Exception e
                             (log/error e "Error checking in-memory database for deletion")
                             ;; we don't want to delete these, so just pretend we could connect
                             true))))]
    (remove can-connect? (filter in-memory? h2-dbs))))

(defn prune-deleted-inmem-databases!
  []
  (when-let [outdated-ids (seq (map :id (deleted-inmem-databases)))]
    (t2/delete! :model/Database :id [:in outdated-ids])))

(defn start!
  []
  (server/start-web-server! #'handler/app)
  (when-not @initialized?
    (init!))
  (when config/is-dev?
    (prune-deleted-inmem-databases!)
    (with-out-str (malli-dev/start!))))

(defn stop!
  []
  (malli-dev/stop!)
  (server/stop-web-server!))

(defn restart!
  []
  (stop!)
  (start!))

(defn ns-unmap-all
  ([]
   (ns-unmap-all *ns*))
  ([a-namespace]
   (doseq [[symb] (ns-interns a-namespace)]
     (ns-unmap a-namespace symb))
   (doseq [[symb varr] (ns-refers a-namespace)
           :when (not= (the-ns (:ns (meta varr)))
                       (the-ns 'clojure.core))]
     (ns-unmap a-namespace symb))))

(defn ns-unalias-all
  ([]
   (ns-unalias-all *ns*))
  ([a-namespace]
   (doseq [[symb] (ns-aliases a-namespace)]
     (ns-unalias a-namespace symb))))

(defmacro require-model
  [model-sym]
  `(require [(symbol (str "metabase.models." (quote ~model-sym))) :as (quote ~model-sym)]))

(defmacro with-permissions
  [permissions & body]
  `(binding [api/*current-user-permissions-set* (delay ~permissions)]
     ~@body))

(defn query-jdbc-db
  {:arglists '([driver sql]            [[driver dataset] sql]
               [driver honeysql-form]  [[driver dataset] honeysql-form]
               [driver [sql & params]] [[driver dataset] [sql & params]])}
  [driver-or-driver+dataset sql-args]
  (let [[driver dataset] (u/one-or-many driver-or-driver+dataset)
        [sql & params]   (if (map? sql-args)
                           (sql/format sql-args)
                           (u/one-or-many sql-args))
        canceled-chan    (a/promise-chan)]
    (try
      (driver/with-driver driver
        (letfn [(thunk []
                  (let [db (mt/db)]
                    (sql-jdbc.execute/do-with-connection-with-options
                     driver
                     db
                     {:session-timezone (qp.timezone/report-timezone-id-if-supported driver db)}
                     (fn [conn]
                       (with-open [stmt (sql-jdbc.execute/prepared-statement driver conn sql params)
                                   rs   (sql-jdbc.execute/execute-prepared-statement! driver stmt)]
                         (let [rsmeta (.getMetaData rs)]
                           {:cols (sql-jdbc.execute/column-metadata driver rsmeta)
                            :rows (reduce conj [] (sql-jdbc.execute/reducible-rows driver rs rsmeta canceled-chan))}))))))]
          (if dataset
            (data.impl/do-with-dataset (data.impl/resolve-dataset-definition *ns* dataset) thunk)
            (thunk))))
      (catch InterruptedException e
        (a/>!! canceled-chan :cancel)
        (throw e)))))

(defn migrate!
  ([]
   (migrate! :up))
  ([direction & [version]]
   (mdb.setup/migrate! (mdb.connection/db-type) (mdb.connection/data-source)
                       direction version)))

(methodical/defmethod t2.connection/do-with-connection :model/Database
  "Support running arbitrary queries against data warehouse DBs for easy REPL debugging. Only works for SQL+JDBC drivers
  right now!
    ;; use Honey SQL
    (t2/query (t2/select-one Database :engine :postgres, :name \"test-data\")
              {:select [:*], :from [:venues]})
    ;; use it with `select`
    (t2/select :conn (t2/select-one Database :engine :postgres, :name \"test-data\")
               \"venues\")
    ;; use it with raw SQL
    (t2/query (t2/select-one Database :engine :postgres, :name \"test-data\")
              \"SELECT * FROM venues;\")"
  [database f]
  (t2.connection/do-with-connection (sql-jdbc.conn/db->pooled-connection-spec database) f))

(methodical/defmethod t2.pipeline/build [#_query-type     :default
                                         #_model          :default
                                         #_resolved-query :mbql]
  [_query-type _model _parsed-args resolved-query]
  resolved-query)

(methodical/defmethod t2.pipeline/compile [#_query-type  :default
                                           #_model       :default
                                           #_built-query :mbql]
  "Run arbitrary MBQL queries. Only works for SQL right now!
    ;; Run a query against a Data warehouse DB
    (t2/query (t2/select-one Database :name \"test-data\")
              (mt/mbql-query venues))
    ;; Run MBQL queries against the application database
    (t2/query (dev/with-app-db (mt/mbql-query core_user {:aggregation [[:min [:get-year $date_joined]]]})))
    =>
    [{:min 2023}]"
  [_query-type _model built-query]
  ;; make sure we use the application database when compiling the query and not something goofy like a connection for a
  ;; Data warehouse DB, if we're using this in combination with a Database as connectable
  (let [{:keys [query params]} (binding [t2.connection/*current-connectable* nil]
                                 (qp/compile built-query))]
    (into [query] params)))

(defn app-db-as-data-warehouse
  []
  (binding [t2.connection/*current-connectable* nil]
    (or (t2/select-one Database :name "Application Database")
        (let [details (#'metabase.db.env/broken-out-details
                       (mdb.connection/db-type)
                       @#'metabase.db.env/env)
              app-db  (first (t2/insert-returning-instances! Database
                                                             {:name    "Application Database"
                                                              :engine  (mdb.connection/db-type)
                                                              :details details}))]
          (sync/sync-database! app-db)
          app-db))))

(defmacro with-app-db
  [& body]
  `(let [db# (app-db-as-data-warehouse)]
     (mt/with-driver (:engine db#)
       (mt/with-db db#
         ~@body))))

(ns dev.debug-qp
  (:require
   [clojure.pprint :as pprint]
   [clojure.string :as str]
   [clojure.walk :as walk]
   [lambdaisland.deep-diff2 :as ddiff]
   [medley.core :as m]
   [metabase.db.query :as mdb.query]
   [metabase.mbql.normalize :as mbql.normalize]
   [metabase.mbql.schema :as mbql.s]
   [metabase.mbql.util :as mbql.u]
   [metabase.models.field :refer [Field]]
   [metabase.models.table :refer [Table]]
   [metabase.query-processor :as qp]
   [metabase.query-processor.reducible :as qp.reducible]
   [metabase.util :as u]
   [toucan2.core :as t2]))

(def ^:private mbql-clause->sort-order
  (into {}
        (map-indexed (fn [i k]
                       [k i]))
        [;; top-level keys
         :database
         :type
         :query
         :native
         ;; inner-query and join keys
         :source-table
         :source-query
         :source-metadata
         :alias
         :joins
         :expressions
         :breakout
         :aggregation
         :condition
         :fields
         :strategy
         :filter
         :order-by
         :page
         :limit]))

(defn- sorted-mbql-query-map []
  ;; stuff in [[mbql-clause->sort-order]] should always get sorted according to that order. Everything else should go at
  ;; the end, with non-namespaced stuff first and namespaced stuff last; otherwise sort alphabetically
  (sorted-map-by (fn [x y]
                   (let [order   (fn [k]
                                   (or (mbql-clause->sort-order k)
                                       (when (and (keyword? k) (namespace k))
                                         Integer/MAX_VALUE)
                                       (dec Integer/MAX_VALUE)))
                         x-order (order x)
                         y-order (order y)]
                     (if (= x-order y-order)
                       (compare (str x) (str y))
                       (compare x-order y-order))))))

(def ^:dynamic *shorten-namespaced-keywords?*
  true)

(defn- alias-for-namespace-in-*ns* [ns-symb]
  (let [a-namespace (find-ns (symbol ns-symb))]
    (some
     (fn [[ns-alias aliased-namespace]]
       (when (= aliased-namespace a-namespace)
         ns-alias))
     (ns-aliases *ns*))))

(defn ->sorted-mbql-query-map
  [query]
  (walk/postwalk
   (fn [form]
     (cond
       (map? form)
       (into (sorted-mbql-query-map) form)
       (and *shorten-namespaced-keywords?*
            (keyword? form)
            (namespace form))
       (if-let [ns-alias (alias-for-namespace-in-*ns* (symbol (namespace form)))]
         (symbol (format "::%s/%s" ns-alias (name form)))
         form)
       :else
       form))
   query))

(defn- field-and-table-name [field-id]
  (let [{field-name :name, table-id :table_id} (t2/select-one [Field :name :table_id] :id field-id)]
    [(t2/select-one-fn :name Table :id table-id) field-name]))

(defn- add-table-id-name [table-id]
  (list 'do
        (symbol (format "#_%s" (pr-str (t2/select-one-fn :name Table :id table-id))))
        table-id))

(defn add-names
  [x]
  (-> (walk/postwalk
       (fn add-names* [form]
         (letfn [(add-name-to-field-id [id]
                   (when id
                     (let [[field-name table-name] (field-and-table-name id)]
                       (symbol (format "#_\"%s.%s\"" field-name table-name)))))
                 (field-id->name-form [field-id]
                   (list 'do (add-name-to-field-id field-id) field-id))]
           (mbql.u/replace form
             [:field (id :guard pos-int?) opts]
             [:field id (add-name-to-field-id id) (cond-> opts
                                                    (pos-int? (:source-field opts))
                                                    (update :source-field field-id->name-form))]
             (m :guard (every-pred map? (comp pos-int? :source-table)))
             (add-names* (update m :source-table add-table-id-name))
             (m :guard (every-pred map? (comp pos-int? :metabase.query-processor.util.add-alias-info/source-table)))
             (add-names* (update m :metabase.query-processor.util.add-alias-info/source-table add-table-id-name))
             (m :guard (every-pred map? (comp pos-int? :fk-field-id)))
             (-> m
                 (update :fk-field-id field-id->name-form)
                 add-names*)
             ;; don't recursively replace the `do` lists above, other we'll get vectors.
             (_ :guard (every-pred list? #(= (first %) 'do)))
             &match)))
       x)
      ->sorted-mbql-query-map))

(def ^:private ^:dynamic *print-full?*     true)
(def ^:private ^:dynamic *print-metadata?* false)
(def ^:private ^:dynamic *print-names?*    true)
(def ^:private ^:dynamic *validate-query?* false)

(defn- remove-metadata
  [x]
  (walk/prewalk
   (fn [form]
     (if (map? form)
       (reduce
        (fn [m k]
          (m/update-existing m k (constantly '...)))
        form
        [:cols :results_metadata :source-metadata])
       form))
   x))

(defn- format-output [x]
  (cond-> x
    (not *print-metadata?*) remove-metadata
    *print-names?*          add-names))

(defn- print-diff [before after]
  (assert (not= before after))
  (ddiff/pretty-print (ddiff/diff before after)
                      ;; the default printer is very (too?) colorful.
                      ;; this is one that strips color except for the diffs:
                      (ddiff/printer {:color-scheme
                                      {:lambdaisland.deep-diff2.printer-impl/deletion  [:red]
                                       :lambdaisland.deep-diff2.printer-impl/insertion [:green]
                                       :lambdaisland.deep-diff2.printer-impl/other     [:white]
                                       :delimiter       nil
                                       :tag             nil
                                       :nil             nil
                                       :boolean         nil
                                       :number          nil
                                       :string          nil
                                       :character       nil
                                       :keyword         nil
                                       :symbol          nil
                                       :function-symbol nil
                                       :class-delimiter nil
                                       :class-name      nil}}))
  (println))

(defn- print-transform-result [before after]
  (when *print-full?*
    (println (u/pprint-to-str 'cyan (format-output after))))
  (print-diff before after))

(defn- print-error [location middleware-var e]
  (println (format "Error %s in %s:\n%s"
                   location
                   middleware-var
                   (u/pprint-to-str 'red (Throwable->map e)))))

(defmulti print-formatted-event
  first)

(defmethod print-formatted-event ::transformed-query
  [[_ middleware-var before after]]
  (println (format "[pre] %s transformed query:" middleware-var))
  (print-transform-result before after))

(defmethod print-formatted-event ::pre-process-query-error
  [[_ middleware-var e]]
  (print-error "pre-processing query" middleware-var e))

(defmethod print-formatted-event ::transformed-metadata
  [[_ middleware-var before after]]
  (println (format "[post] %s transformed metadata:" middleware-var))
  (print-transform-result before after))

(defmethod print-formatted-event ::post-process-metadata-error
  [[_ middleware-var e]]
  (print-error "post-processing result metadata" middleware-var e))

(defmethod print-formatted-event ::post-process-result-error
  [[_ middleware-var e]]
  (print-error "post-processing result" middleware-var e))

(defmethod print-formatted-event ::transformed-result
  [[_ middleware-var before after]]
  (println (format "[post] %s transformed result:" middleware-var))
  (print-transform-result before after))

(defmethod print-formatted-event ::error-reduce-row
  [[_ middleware-var e]]
  (print-error "reducing row" middleware-var e))

(defmethod print-formatted-event ::transformed-row
  [[_ middleware-var before after]]
  (println (format "[post] %s transformed row" middleware-var))
  (print-transform-result before after))

(def ^:private ^:dynamic *printer* print-formatted-event)

(defn- debug-query-changes [middleware-var middleware]
  (fn [next-middleware]
    (fn [query-before rff context]
      (try
        ((middleware
          (fn [query-after rff context]
            (when-not (= query-before query-after)
              (*printer* [::transformed-query middleware-var query-before query-after]))
            (when *validate-query?*
              (try
                (mbql.s/validate-query query-after)
                (catch Throwable e
                  (when (::our-error? (ex-data e))
                    (throw e))
                  (throw (ex-info (format "%s middleware produced invalid query" middleware-var)
                                  {::our-error? true
                                   :middleware  middleware-var
                                   :before      query-before
                                   :query       query-after}
                                  e)))))
            (next-middleware query-after rff context)))
         query-before rff context)
        (catch Throwable e
          (when (::our-error? (ex-data e))
            (throw e))
          (*printer* [::pre-process-query-error middleware-var e])
          (throw (ex-info "Error pre-processing query"
                          {::our-error? true
                           :middleware  middleware-var
                           :query       query-before}
                          e)))))))

(defn- debug-rffs [middleware-var middleware before-rff-xform after-rff-xform]
  (fn [next-middleware]
    (fn [query rff-after context]
      ((middleware
        (fn [query rff-before context]
          (next-middleware query (before-rff-xform rff-before) context)))
       query (after-rff-xform rff-after) context))))

(defn- debug-metadata-changes [middleware-var middleware]
  (let [before (atom nil)]
    (debug-rffs
     middleware-var
     middleware
     (fn before-rff-xform [rff]
       (fn [metadata-before]
         (reset! before metadata-before)
         (try
           (rff metadata-before)
           (catch Throwable e
             (when (::our-error? (ex-data e))
               (throw e))
             (*printer* [::post-process-metadata-error middleware-var e])
             (throw (ex-info "Error post-processing result metadata"
                             {::our-error? true
                              :middleware  middleware-var
                              :metadata    metadata-before}
                             e))))))
     (fn after-rff-xform [rff]
       (fn [metadata-after]
         (when-not (= @before metadata-after)
           (*printer* [::transformed-metadata middleware-var @before metadata-after]))
         (rff metadata-after))))))

(defn- debug-rfs [middleware-var middleware before-xform after-xform]
  (debug-rffs
   middleware-var
   middleware
   (fn before-rff-xform [rff]
     (fn [metadata]
       (let [rf (rff metadata)]
         (before-xform rf))))
   (fn after-rff-xform [rff]
     (fn [metadata]
       (let [rf (rff metadata)]
         (after-xform rf))))))

(defn- debug-result-changes [middleware-var middleware]
  (let [before (atom nil)]
    (debug-rfs
     middleware-var
     middleware
     (fn before-xform [rf]
       (fn
         ([] (rf))
         ([result]
          (reset! before result)
          (try
            (rf result)
            (catch Throwable e
              (when (::our-error? (ex-data e))
                (throw e))
              (*printer* [::post-process-result-error middleware-var e])
              (throw (ex-info "Error post-processing result"
                              {::our-error? true
                               :middleware  middleware-var
                               :result      result}
                              e)))))
         ([result row] (rf result row))))
     (fn after-xform [rf]
       (fn
         ([] (rf))
         ([result]
          (when-not (= @before result)
            (*printer* [::transformed-result middleware-var @before result]))
          (rf result))
         ([result row] (rf result row)))))))

(defn- debug-row-changes [middleware-var middleware]
  (let [before (atom nil)]
    (debug-rfs
     middleware-var
     middleware
     (fn before-xform [rf]
       (fn
         ([] (rf))
         ([result]
          (rf result))
         ([result row]
          (reset! before row)
          (try
            (rf result row)
            (catch Throwable e
              (when (::our-error? (ex-data e))
                (throw e))
              (*printer* [::error-reduce-row middleware-var e])
              (throw (ex-info "Error reducing row"
                              {::our-error? true
                               :middleware  middleware-var
                               :result      result
                               :row         row}
                              e)))))))
     (fn after-xform [rf]
       (fn
         ([] (rf))
         ([result]
          (rf result))
         ([result row]
          (when-not (= @before row)
            (*printer* [::transformed-row @before row]))
          (rf result row)))))))

(defn- default-debug-middleware
  []
  (into
   []
   (comp cat (keep identity))
   [@#'qp/execution-middleware
    @#'qp/compile-middleware
    @#'qp/post-processing-middleware
    ;; Normally, pre-processing middleware are applied to the query left-to-right, but in debug mode we convert each
    ;; one into a transducing middleware and compose them, which causes them to be applied right-to-left. So we need
    ;; to reverse the order here.
    (reverse @#'qp/pre-processing-middleware)
    @#'qp/around-middleware]))

(defn- alter-pre-processing-middleware
  [middleware]
  (fn [qp-or-query]
    (if (map? qp-or-query)
      ;; If we're passed a map, this means the middleware var is still being called on a query directly. This happens
      ;; if pre-processing middleware calls other pre-processing middleware, such as [[upgrade-field-literals]] which
      ;; calls [[resolve-fields]]. Fallback to the original middleware function in this case.
      (middleware qp-or-query)
      (fn [query rff context]
        (qp-or-query
         (middleware query)
         rff
         context)))))

(defn- alter-post-processing-middleware
  [middleware]
  (fn [qp]
    (fn [query rff context]
      (qp query (middleware query rff) context))))

(defn- with-altered-middleware-fn
  [f]
  (let [pre-processing-middleware-vars  @#'qp/pre-processing-middleware
        post-processing-middleware-vars @#'qp/post-processing-middleware
        pre-processing-original-fns     (zipmap pre-processing-middleware-vars
                                                (map deref pre-processing-middleware-vars))
        post-processing-original-fns    (zipmap post-processing-middleware-vars
                                                (map deref post-processing-middleware-vars))]
    (try
      (mapv #(alter-var-root % alter-pre-processing-middleware) pre-processing-middleware-vars)
      (mapv #(alter-var-root % alter-post-processing-middleware) post-processing-middleware-vars)
      (f)
      (finally
        (mapv (fn [[middleware-var middleware-fn]]
                (alter-var-root middleware-var (constantly middleware-fn)))
              (merge pre-processing-original-fns post-processing-original-fns))))))

(defmacro ^:private with-altered-middleware
  [& body]
  `(with-altered-middleware-fn (fn [] ~@body)))

(defn process-query-debug
  [query & {:keys [print-full? print-metadata? print-names? validate-query? printer context]
            :or   {print-full? true, print-metadata? false, print-names? true, validate-query? false
                   printer print-formatted-event}}]
  (binding [*print-full?*               print-full?
            *print-metadata?*           print-metadata?
            *print-names?*              print-names?
            *validate-query?*           validate-query?
            *printer*                   printer
            pprint/*print-right-margin* 80]
    (with-altered-middleware
      (let [middleware (for [middleware-var (default-debug-middleware)
                             :when          middleware-var]
                         (->> middleware-var
                              (debug-query-changes middleware-var)
                              (debug-metadata-changes middleware-var)
                              (debug-result-changes middleware-var)
                              (debug-row-changes middleware-var)))
            qp         (qp.reducible/sync-qp (#'qp/base-qp middleware))]
        (if context
          (qp query context)
          (qp query))))))

(defn- strip-$ [coll]
  (into []
        (map (fn [x] (if (= x ::$) ::no-$ x)))
        coll))

(defn- can-symbolize? [x]
  (mbql.u/match-one x
    (_ :guard string?)
    (not (re-find #"\s+" x))
    [:field (id :guard pos-int?) nil]
    (every? can-symbolize? (field-and-table-name id))
    [:field (field-name :guard string?) (opts :guard #(= (set (keys %)) #{:base-type}))]
    (can-symbolize? field-name)
    [:field _ (opts :guard :join-alias)]
    (and (can-symbolize? (:join-alias opts))
         (can-symbolize? (mbql.u/update-field-options &match dissoc :join-alias)))
    [:field _ (opts :guard :temporal-unit)]
    (and (can-symbolize? (name (:temporal-unit opts)))
         (can-symbolize? (mbql.u/update-field-options &match dissoc :temporal-unit)))
    [:field _ (opts :guard :source-field)]
    (let [source-field-id (:source-field opts)]
      (and (can-symbolize? [:field source-field-id nil])
           (can-symbolize? (mbql.u/update-field-options &match dissoc :source-field))))
    _
    false))

(defn- expand [form table]
  (try
    (mbql.u/replace form
      ([:field (id :guard pos-int?) nil] :guard can-symbolize?)
      (let [[table-name field-name] (field-and-table-name id)
            field-name              (some-> field-name u/lower-case-en)
            table-name              (some-> table-name u/lower-case-en)]
        (if (= table-name table)
          [::$ field-name]
          [::$ table-name field-name]))
      ([:field (field-name :guard string?) (opts :guard #(= (set (keys %)) #{:base-type}))] :guard can-symbolize?)
      [::* field-name (name (:base-type opts))]
      ([:field _ (opts :guard :temporal-unit)] :guard can-symbolize?)
      (let [without-unit (mbql.u/update-field-options &match dissoc :temporal-unit)
            expansion    (expand without-unit table)]
        [::! (name (:temporal-unit opts)) (strip-$ expansion)])
      ([:field _ (opts :guard :source-field)] :guard can-symbolize?)
      (let [without-source-field   (mbql.u/update-field-options &match dissoc :source-field)
            expansion              (expand without-source-field table)
            source-as-field-clause [:field (:source-field opts) nil]
            source-expansion       (expand source-as-field-clause table)]
        [::-> source-expansion expansion])
      ([:field _ (opts :guard :join-alias)] :guard can-symbolize?)
      (let [without-join-alias (mbql.u/update-field-options &match dissoc :join-alias)
            expansion          (expand without-join-alias table)]
        [::& (:join-alias opts) expansion])
      [:field (id :guard pos-int?) opts]
      (let [without-opts [:field id nil]
            expansion    (expand without-opts table)]
        (if (= expansion without-opts)
          &match
          [:field [::% (strip-$ expansion)] opts]))
      (m :guard (every-pred map? (comp pos-int? :source-table)))
      (-> (update m :source-table (fn [table-id]
                                    [::$$ (some-> (t2/select-one-fn :name Table :id table-id) u/lower-case-en)]))
          (expand table))
      (m :guard (every-pred map? (comp pos-int? :fk-field-id)))
      (-> (update m :fk-field-id (fn [fk-field-id]
                                   (let [[table-name field-name] (field-and-table-name fk-field-id)
                                         field-name              (some-> field-name u/lower-case-en)
                                         table-name              (some-> table-name u/lower-case-en)]
                                     (if (= table-name table)
                                       [::% field-name]
                                       [::% table-name field-name]))))
          (expand table)))
    (catch Throwable e
      (throw (ex-info (format "Error expanding %s: %s" (pr-str form) (ex-message e))
                      {:form form, :table table}
                      e)))))

(defn- no-$ [x]
  (mbql.u/replace x [::$ & args] (into [::no-$] args)))

(defn- symbolize [form]
  (mbql.u/replace form
    [::-> x y]
    (symbol (format "%s->%s" (symbolize x) (str/replace (symbolize y) #"^\$" )))
    [::no-$ & args]
    (str/join \. args)
    [(qualifier :guard #{::$ ::& ::! ::%}) & args]
    (symbol (str (name qualifier) (str/join \. (symbolize (no-$ args)))))
    [::* field-name base-type]
    (symbol (format "*%s/%s" field-name base-type))
    [::$$ table-name]
    (symbol (format "$$%s" table-name))))

(defn- query-table-name [{:keys [source-table source-query], :as inner-query}]
  (cond
    (pos-int? source-table)
    (u/lower-case-en (or (t2/select-one-fn :name Table :id source-table)
                         (throw (ex-info (format "Table %d does not exist!" source-table)
                                         {:source-table source-table, :inner-query inner-query}))))
    source-query
    (recur source-query)))

(defn to-mbql-shorthand
  ([query]
   (let [query (mbql.normalize/normalize query)]
     (to-mbql-shorthand query (query-table-name (:query query)))))
  ([query table-name]
   (let [symbolized (-> query (expand table-name) symbolize ->sorted-mbql-query-map)
         table-symb (some-> table-name symbol)]
     (if (:query symbolized)
       (list 'mt/mbql-query table-symb (cond-> (:query symbolized)
                                         table-name (dissoc :source-table)))
       (list 'mt/$ids table-symb symbolized)))))

(defn expand-symbolize [x]
  (-> x (expand "orders") symbolize))

(defn pprint-sql
  [driver sql]
  #_{:clj-kondo/ignore [:discouraged-var]}
   (println (mdb.query/format-sql sql driver)))

(ns  dev.debug-qp-test
  (:require [clojure.test :refer :all]
            [dev.debug-qp :as debug-qp]
            [metabase.test :as mt]))

(deftest add-names-test
  (testing "Joins"
    (is (= [{:strategy     :left-join
             :alias        "CATEGORIES__via__CATEGORY_ID"
             :condition    [:=
                            [:field
                             (mt/id :venues :category_id)
                             (symbol "#_\"VENUES.CATEGORY_ID\)
                             nil]
                            [:field
                             (mt/id :categories :id)
                             (symbol "#_\"CATEGORIES.ID\)
                             {:join-alias "CATEGORIES__via__CATEGORY_ID"}]]
             :source-table (list 'do (symbol "#_\"CATEGORIES\) (mt/id :categories))
             :fk-field-id  (list 'do (symbol "#_\"VENUES.CATEGORY_ID\) (mt/id :venues :category_id))}]
           (debug-qp/add-names
            [{:strategy     :left-join
              :alias        "CATEGORIES__via__CATEGORY_ID"
              :condition    [:=
                             [:field (mt/id :venues :category_id) nil]
                             [:field (mt/id :categories :id) {:join-alias "CATEGORIES__via__CATEGORY_ID"}]]
              :source-table (mt/id :categories)
              :fk-field-id  (mt/id :venues :category_id)}])))))

(deftest to-mbql-shorthand-test
  (mt/dataset test-data
    (testing "Normal Field ID clause"
      (is (= '$user_id
             (debug-qp/expand-symbolize [:field (mt/id :orders :user_id) nil])))
      (is (= '$products.id
             (debug-qp/expand-symbolize [:field (mt/id :products :id) nil]))))
    (testing "Field literal name"
      (is (= '*wow/Text
             (debug-qp/expand-symbolize [:field "wow" {:base-type :type/Text}])))
      (is (= [:field "w o w" {:base-type :type/Text}]
             (debug-qp/expand-symbolize [:field "w o w" {:base-type :type/Text}]))))
    (testing "Field with join alias"
      (is (= '&P.people.source
             (debug-qp/expand-symbolize [:field (mt/id :people :source) {:join-alias "P"}])))
      (is (= [:field '%people.id {:join-alias "People - User"}]
             (debug-qp/expand-symbolize [:field (mt/id :people :id) {:join-alias "People - User"}])))
      (is (= '&Q.*ID/BigInteger
             (debug-qp/expand-symbolize [:field "ID" {:base-type :type/BigInteger, :join-alias "Q"}]))))
    (testing "Field with source-field"
      (is (= '$product_id->products.id
             (debug-qp/expand-symbolize [:field (mt/id :products :id) {:source-field (mt/id :orders :product_id)}])))
      (is (= '$product_id->*wow/Text
             (debug-qp/expand-symbolize [:field "wow" {:base-type :type/Text, :source-field (mt/id :orders :product_id)}]))))
    (testing "Binned field - no expansion (%id only)"
      (is (= [:field '%people.source {:binning {:strategy :default}}]
             (debug-qp/expand-symbolize [:field (mt/id :people :source) {:binning {:strategy :default}}]))))
    (testing "Field with temporal unit"
      (is (= '!default.created_at
             (debug-qp/expand-symbolize [:field (mt/id :orders :created_at) {:temporal-unit :default}]))))
    (testing "Field with join alias AND temporal unit"
      (is (= '!default.&P1.created_at
             (debug-qp/expand-symbolize [:field (mt/id :orders :created_at) {:temporal-unit :default, :join-alias "P1"}]))))
    (testing "source table"
      (is (= '(mt/mbql-query orders
                {:joins [{:source-table $$people}]})
             (debug-qp/to-mbql-shorthand
              {:database (mt/id)
               :type     :query
               :query    {:source-table (mt/id :orders)
                          :joins        [{:source-table (mt/id :people)}]}}))))))

(deftest to-mbql-shorthand-joins-test
  (testing :fk-field-id
    (is (= '(mt/$ids venues
              [{:strategy     :left-join
                :alias        "CATEGORIES__via__CATEGORY_ID"
                :condition    [:= $category_id &CATEGORIES__via__CATEGORY_ID.categories.id]
                :source-table $$categories
                :fk-field-id  %category_id}])
           (debug-qp/to-mbql-shorthand
            [{:strategy     :left-join
              :alias        "CATEGORIES__via__CATEGORY_ID"
              :condition    [:=
                             [:field (mt/id :venues :category_id) nil]
                             [:field (mt/id :categories :id) {:join-alias "CATEGORIES__via__CATEGORY_ID"}]]
              :source-table (mt/id :categories)
              :fk-field-id  (mt/id :venues :category_id)}]
            "venues")))))

(ns dev.explain
  (:require
   [clojure.string :as str]
   [honey.sql :as sql]
   [toucan2.core :as t2]))

(defn explain-query
  ([queryable]
   (explain-query queryable false))
  ([queryable analyze?]
   (->> (t2/query
         (str/join
          " "
          (remove nil? ["EXPLAIN"
                        (when analyze? "ANALYZE")
                        "(" (if (map? queryable) (first (sql/format queryable {:inline true})) queryable) ")"])))
        (map #(get % (keyword "query plan"))))))

(ns dev.fe-helpers)

(defn redux-state
  []
  (let [root  (js/document.querySelector "#root")
        store (.. root -_reactRootContainer -_internalRoot -current -child -memoizedProps -store)]
    (.getState store)))

(defn current-card
  []
  (.. (redux-state) -qb -card))

(defn current-legacy-query-js
  []
  (.-dataset_query (current-card)))

(defn current-query
  []
  (.-__MLv2_query js/window))

(ns dev.h2-shell
  (:require [environ.core :as env]
            [metabase.db.data-source :as mdb.data-source]
            [metabase.db.env :as mdb.env]))

(comment mdb.data-source/keep-me)

(defn shell
  [& _args]
  ;; Force the DB to use h2 regardless of what's actually in the env vars for Java properties
  (alter-var-root #'env/env assoc :mb-db-type "h2")
  (require 'metabase.db.env :reload)
  (org.h2.tools.Shell/main
   (into-array
    String
    ["-url" (let [^metabase.db.data_source.DataSource data-source mdb.env/data-source
                  url                                             (.url data-source)]
              (println "Connecting to database at URL" url)
              url)])))

(ns dev.liquibase
  (:require [clojure.string :as str]
            [colorize.core :as colorize]
            [metabase.db.data-source :as mdb.data-source]
            [metabase.db.env :as mdb.env]))

(comment mdb.data-source/keep-me)

(defn -main
  [& args]
  (let [args (if (empty? args)
               ["help"]
               args)
        args (into ["--changeLogFile=resources/migrations/000_migrations.yaml"]
                   (comp cat
                         (filter seq))
                   (let [^metabase.db.data_source.DataSource data-source mdb.env/data-source
                         ^java.util.Properties properties                (.properties data-source)]
                     [(when-let [user (some-> properties (.get "user"))]
                        ["--username" user])
                      (when-let [password (some-> properties (.get "password"))]
                        ["--password" password])
                      ["--url" (.url data-source)]
                      (map str args)]))]
    (println (colorize/green (str/join " " (cons "liquibase" (map pr-str args)))))
    ;; use reflection here instead of static method calls because `liquibase.integration.commandline.Main` fails to load
    ;; without having the `logback` dependency available. We add this as `:extra-deps` for the `:liquibase` profile. We
    ;; don't want other stuff like the linters to choke here tho.
    (let [klass  (Class/forName "liquibase.integration.commandline.Main")
          method (.getMethod klass "main" (into-array Class [(Class/forName "[Ljava.lang.String;")]))]
      (.invoke method klass ^"[Ljava.lang.Object" (into-array Object [(into-array String args)])))))

(ns dev.model-tracking
  (:require
   [clojure.pprint :as pprint]
   [metabase.util :as u]
   [methodical.core :as m]
   [toucan2.core :as t2]
   [toucan2.model :as t2.model]
   [toucan2.tools.before-delete :as t2.before-delete]
   [toucan2.tools.before-insert :as t2.before-insert]
   [toucan2.tools.before-update :as t2.before-update]
   [toucan2.util :as t2.util]))

(def changes*
  (atom {}))

(def ^:private tracked-models (atom #{}))

(defn on-change
  [path change-info]
  (println (u/colorize :magenta :new-change) (u/colorize :magenta path))
  (pprint/pprint change-info))

(defn- clean-change
  [change]
  (dissoc change :updated_at :created_at))

(defn- new-change
  [model action row-or-instance]
  (let [model       (t2/resolve-model model)
        change-info (->> (case action
                           :update
                           (into {} (t2/changes row-or-instance))
                           (into {} row-or-instance))
                        clean-change)
        path       [(t2/table-name model) action]]
    ;; ideally this should be debug, but for some reasons this doesn't get logged
    (on-change path change-info)
    (swap! changes* update-in path concat [change-info])))

(defn- new-change-thunk
  [model action]
  (fn [_model row]
    (new-change model action row)
    row))

(def ^:private hook+aux-method+action+deriveable
  [;; will be better if we could use after-insert to get the inserted id, but toucan2 doesn't define a multimethod for after-insert
   [#'t2.before-insert/before-insert :after :insert ::t2.before-insert/before-insert]
   [#'t2.before-update/before-update :after :update ::t2.before-update/before-update]
   ;; we do :before aux-method instead of :after for delete bacause the after method has input is number of affected rows
   [#'t2.before-delete/before-delete :before :delete ::t2.before-delete/before-delete]])

(defn- track-one!
  [model]
  (doseq [[hook aux-method action deriveable] hook+aux-method+action+deriveable]
    (when-not (m/primary-method @hook model)
      ;; aux-method will not be triggered if there isn't a primary method
      (t2.util/maybe-derive model deriveable)
      (m/add-primary-method! hook model (fn [_ _model row] row)))
    (m/add-aux-method-with-unique-key! hook aux-method model (new-change-thunk model action) ::tracking)))

(defn track!
  [& models]
  (doseq [model (map t2.model/resolve-model models)]
    (track-one! model)
    (swap! tracked-models conj model)))

(defn- untrack-one!
  [model]
  (doseq [[hook aux-method _action] hook+aux-method+action+deriveable]
    (m/remove-aux-method-with-unique-key! hook aux-method model ::tracking)
    (swap! tracked-models disj model)))

(defn untrack!
  [& models]
  (doseq [model (map t2.model/resolve-model models)]
    (untrack-one! model)))

(defn reset-changes!
  []
  (reset! changes* {}))

(defn untrack-all!
  []
  (reset-changes!)
  (apply untrack! @tracked-models)
  (reset! tracked-models #{}))

(defn changes
  []
  @changes*)

(ns dev.model-tracking-test
  (:require
   [clojure.test :refer :all]
   [dev.model-tracking :as model-tracking]
   [metabase.models :refer [Collection]]
   [metabase.test :as mt]
   [toucan2.core :as t2]))

(use-fixtures :each (fn [thunk]
                      (model-tracking/untrack-all!)
                      (thunk)))

(deftest e2e-test
  (mt/with-model-cleanup [Collection]
    ;; setup
    (model-tracking/track! 'Collection)
    (testing "insert"
      (t2/insert! Collection {:name "Test tracking" :description "My awesome collection"})
      (testing "should be tracked"
        (is (=? [{:name  "Test tracking"
                  :description "My awesome collection"}]
                (get-in (model-tracking/changes) [:collection :insert]))))
      (testing "should take affects"
        (is (= 1 (t2/count Collection :name "Test tracking")))))
    (testing "update"
      (t2/update! Collection {:name "Test tracking"} {:description "Amazing collection"})
      (testing "changes should be tracked"
        (is (= [{:description "Amazing collection"}]
               (get-in (model-tracking/changes) [:collection :update]))))
      (testing "should take affects"
        (is (= "Amazing collection" (t2/select-one-fn :description Collection :name "Test tracking")))))
    (testing "delete"
      (let [coll-id (t2/select-one-pk Collection :name "Test tracking")]
        (t2/delete! Collection coll-id)
        (testing "should be tracked"
          (is (=? [{:description "Amazing collection"
                    :name  "Test tracking",
                    :id    coll-id}]
                  (get-in (model-tracking/changes) [:collection :delete]))))
        (testing "should take affects"
          (is (nil? (t2/select-one Collection :id coll-id))))))
    (testing "untrack should stop all tracking for"
      (model-tracking/untrack-all!)
      (testing "insert"
        (t2/insert! Collection {:name "Test tracking" :description "My awesome collection"})
        (testing "changes not should be tracked"
          (is (empty? (model-tracking/changes))))
        (testing "should take affects"
          (is (= 1 (t2/count Collection :name "Test tracking")))))
      (testing "update"
        (t2/update! Collection {:name "Test tracking"} {:description "Amazing collection"})
        (testing "changes not should be tracked"
          (is (empty? (model-tracking/changes))))
        (testing "should take affects"
          (is (= "Amazing collection" (t2/select-one-fn :description Collection :name "Test tracking")))))
      (testing "delete"
        (let [coll-id (t2/select-one-pk Collection :name "Test tracking")]
          (t2/delete! Collection coll-id)
          (testing "changes not should be tracked"
            (is (empty? (model-tracking/changes))))
          (testing "should take affects"
            (is (nil? (t2/select-one Collection :id coll-id)))))))))

(ns dev.portal
  (:require [portal.api :as p]))

(defonce
  p
  (p/open {:port 5678}))

(add-tap #'p/submit)

(doseq [f [#'reverse #'vec]]
  (p/register! f))

(defn unfreeze
  []
  (p/open p))

(defn send-log
  ([value] (send-log value nil))
  ([value {:keys [level ns line column time]
           :or {level  :info
                ns     (ns-name *ns*)
                line   -1
                column -1
                time   (java.util.Date.)}}]
   (tap> {:result value
          :level  level
          :ns     ns
          :line   line
          :column column
          :time   time})))

(defmacro log
  [value & [opts]]
  `(send-log ~value ~(merge (meta &form)
                            {:ns (list 'quote (ns-name *ns*))}
                            opts)))

(defn debug-qp-log
  [[tag middleware-var before after :as event]]
  (if (#{:dev.debug-qp/transformed-query, :dev.debug-qp/transformed-metadata
         :dev.debug-qp/transformed-result, :dev.debug-qp/transformed-row}
       tag)
    (send-log (with-meta [before after]
                {:portal.viewer/default :portal.viewer/diff})
              (update (meta middleware-var) :ns #(.name %)))
    (send-log event)))

(ns dev.render-png
  (:require
    [clojure.data.csv :as csv]
    [clojure.java.io :as io]
    [clojure.java.shell :as sh]
    [hiccup.core :as hiccup]
    [metabase.email.messages :as messages]
    [metabase.models :refer [Card]]
    [metabase.models.card :as card]
    [metabase.pulse :as pulse]
    [metabase.pulse.markdown :as markdown]
    [metabase.pulse.render :as render]
    [metabase.pulse.render.image-bundle :as img]
    [metabase.pulse.render.png :as png]
    [metabase.pulse.render.style :as style]
    [metabase.query-processor :as qp]
    [metabase.test :as mt]
    [toucan2.core :as t2])
  (:import (java.io File)))

(set! *warn-on-reflection* true)

(defn- os
  []
  (condp
   #(<= 0 (.indexOf ^String %2 ^String %1))
   (.toLowerCase (System/getProperty "os.name"))
    "win" :win
    "mac" :mac
    "nix" :unix
    "nux" :unix
    nil))

(defn- open
  [f]
  (let [f (io/file f)]
    ;; There's an 'open' method in java.awt.Desktop but it hangs on Windows
    ;; using Clojure Box and turns the process into a GUI process on Max OS X.
    ;; Maybe it's ok for Linux?
    (condp = (os)
      :mac  (sh/sh "open" (str f))
      :win  (sh/sh "cmd" (str "/c start " (-> f .toURI .toURL str)))
      :unix (sh/sh "xdg-open" (str f)))
    nil))

(defn render-card-to-png
  [card-id]
  (let [{:keys [dataset_query result_metadata dataset] :as card} (t2/select-one card/Card :id card-id)
        query-results (qp/process-query
                        (cond-> dataset_query
                          dataset
                          (assoc-in [:info :metadata/dataset-metadata] result_metadata)))
        png-bytes     (render/render-pulse-card-to-png (pulse/defaulted-timezone card)
                                                       card
                                                       query-results
                                                       1000)
        tmp-file      (File/createTempFile "card-png" ".png")]
    (with-open [w (java.io.FileOutputStream. tmp-file)]
      (.write w ^bytes png-bytes))
    (.deleteOnExit tmp-file)
    (open tmp-file)))

(defn render-pulse-card
  [card-id]
  (let [{:keys [dataset_query] :as card} (t2/select-one card/Card :id card-id)
        query-results (qp/process-query dataset_query)]
    (render/render-pulse-card
     :inline (pulse/defaulted-timezone card)
     card
     nil
     query-results)))

(defn open-hiccup-as-html
  [hiccup]
  (let [html-str (hiccup/html hiccup)
        tmp-file (File/createTempFile "card-html" ".html")]
    (with-open [w (io/writer tmp-file)]
      (.write w ^String html-str))
    (.deleteOnExit tmp-file)
    (open tmp-file)))

(def ^:private execute-dashboard #'pulse/execute-dashboard)

(defn render-dashboard-to-pngs
  [dashboard-id]
  (let [user              (t2/select-one :model/User)
        dashboard         (t2/select-one :model/Dashboard :id dashboard-id)
        dashboard-results (execute-dashboard {:creator_id (:id user)} dashboard)]
    (doseq [{:keys [card dashcard result] :as dashboard-result} dashboard-results]
      (let [render    (if card
                        (render/render-pulse-card :inline (pulse/defaulted-timezone card) card dashcard result)
                        {:content     [:div {:style (style/style {:font-family             "Lato"
                                                                  :font-size               "0.875em"
                                                                  :font-weight             "400"
                                                                  :font-style              "normal"
                                                                  :color                   "#4c5773"
                                                                  :-moz-osx-font-smoothing "grayscale"})}
                                       (markdown/process-markdown (:text dashboard-result) :html)]
                         :attachments nil})
            png-bytes (-> render (png/render-html-to-png 1000))
            tmp-file  (java.io.File/createTempFile "card-png" ".png")]
        (with-open [w (java.io.FileOutputStream. tmp-file)]
          (.write w ^bytes png-bytes))
        (.deleteOnExit tmp-file)
        (open tmp-file)))))

(def ^:private table-style-map
  {:border          "1px solid black"
   :border-collapse "collapse"
   :padding         "5px"})

(def ^:private table-style
  (style/style table-style-map))

(def ^:private csv-row-limit 10)

(defn- csv-to-html-table [csv-string]
  (let [rows (csv/read-csv csv-string)]
    [:table {:style table-style}
     (for [row (take (inc csv-row-limit) rows)] ;; inc row-limit to include the header and the expected # of rows
       [:tr {:style table-style}
        (for [cell row]
          [:td {:style table-style} cell])])]))

(def ^:private result-attachment #'messages/result-attachment)

(defn- render-csv-for-dashcard
  [part]
  (-> part
      (assoc-in [:card :include_csv] true)
      result-attachment
      first
      :content
      slurp
      csv-to-html-table))

(defn- render-one-dashcard
  [{:keys [card dashcard result] :as dashboard-result}]
  (letfn [(cellfn [content]
            [:td {:style (style/style (merge table-style-map {:max-width "400px"}))}
             content])]
    (if card
      (let [base-render (render/render-pulse-card :inline (pulse/defaulted-timezone card) card dashcard result)
            html-src    (-> base-render :content)
            img-src     (-> base-render
                            (png/render-html-to-png 1200)
                            img/render-img-data-uri)
            csv-src (render-csv-for-dashcard dashboard-result)]
        [:tr
         (cellfn (:name card))
         (cellfn [:img {:style (style/style {:max-width "400px"}) :src img-src}])
         (cellfn html-src)
         (cellfn csv-src)])
      [:tr
       (cellfn nil)
       (cellfn
        [:div {:style (style/style {:font-family             "Lato"
                                    :font-size               "13px" #_ "0.875em"
                                    :font-weight             "400"
                                    :font-style              "normal"
                                    :color                   "#4c5773"
                                    :-moz-osx-font-smoothing "grayscale"})}
         (markdown/process-markdown (:text dashboard-result) :html)])
       (cellfn nil)])))

(defn render-dashboard-to-hiccup
  [dashboard-id]
  (let [user              (t2/select-one :model/User)
        dashboard         (t2/select-one :model/Dashboard :id dashboard-id)
        dashboard-results (execute-dashboard {:creator_id (:id user)} dashboard)
        render            (->> (map render-one-dashcard (map #(assoc % :dashboard-id dashboard-id) dashboard-results))
                               (into [[:tr
                                       [:th {:style (style/style table-style-map)} "Card Name"]
                                       [:th {:style (style/style table-style-map)} "PNG"]
                                       [:th {:style (style/style table-style-map)} "HTML"]
                                       [:th {:style (style/style table-style-map)} "CSV"]]])
                               (into [:table {:style (style/style table-style-map)}]))]
    render))

(defn render-dashboard-to-html
  [dashboard-id]
  (hiccup/html (render-dashboard-to-hiccup dashboard-id)))

(defn render-dashboard-to-html-and-open
  [dashboard-id]
  (let [html-str (render-dashboard-to-html dashboard-id)
        tmp-file (File/createTempFile "card-html" ".html")]
    (with-open [w (io/writer tmp-file)]
      (.write w ^String html-str))
    (.deleteOnExit tmp-file)
    (open tmp-file)))

(comment
  ;; This form has 3 cards:
  ;; - A plain old question
  ;; - A model with user defined metadata
  ;; - A question based on that model
  ;;
  ;; The expected rendered results should be:
  ;; - The plain question will not have custom formatting applied
  ;; - The model and derived query will have custom formatting applied
  (mt/dataset sample-dataset
    (mt/with-temp [Card {base-card-id :id}
                   {:dataset_query {:database (mt/id)
                                    :type     :query
                                    :query    {:source-table (mt/id :orders)
                                               :expressions  {"Tax Rate" [:/
                                                                          [:field (mt/id :orders :tax) {:base-type :type/Float}]
                                                                          [:field (mt/id :orders :total) {:base-type :type/Float}]]},
                                               :fields       [[:field (mt/id :orders :tax) {:base-type :type/Float}]
                                                              [:field (mt/id :orders :total) {:base-type :type/Float}]
                                                              [:expression "Tax Rate"]]
                                               :limit        10}}}
                   Card {model-card-id :id} {:dataset         true
                                             :dataset_query   {:type     :query
                                                               :database (mt/id)
                                                               :query    {:source-table (format "card__%s" base-card-id)}}
                                             :result_metadata [{:name         "TAX"
                                                                :display_name "Tax"
                                                                :base_type    :type/Float}
                                                               {:name         "TOTAL"
                                                                :display_name "Total"
                                                                :base_type    :type/Float}
                                                               {:name          "Tax Rate"
                                                                :display_name  "Tax Rate"
                                                                :base_type     :type/Float
                                                                :semantic_type :type/Percentage
                                                                :field_ref     [:field "Tax Rate" {:base-type :type/Float}]}]}
                   Card {question-card-id :id} {:dataset_query {:type     :query
                                                                :database (mt/id)
                                                                :query    {:source-table (format "card__%s" model-card-id)}}}]
      (render-card-to-png base-card-id)
      (render-card-to-png model-card-id)
      (render-card-to-png question-card-id))))

(ns user
  (:require
   [environ.core :as env]
   [humane-are.core :as humane-are]
   [mb.hawk.assert-exprs]
   [metabase.bootstrap]
   [metabase.test-runner.assert-exprs]
   [pjstadig.humane-test-output :as humane-test-output]))

(when-not (= (env/env :mb-run-mode) "test")
  (humane-test-output/activate!))

(humane-are/install!)

(comment metabase.bootstrap/keep-me
         ;; make sure stuff like `=?` and what not are loaded
         mb.hawk.assert-exprs/keep-me
         metabase.test-runner.assert-exprs/keep-me)

(defn dev
  []
  (require 'dev)
  (in-ns 'dev)
  :loaded)

(ns metabase.lib.metadata.cached-provider
  (:require
   [clojure.set :as set]
   [metabase.lib.metadata :as lib.metadata]
   [metabase.lib.metadata.protocols :as lib.metadata.protocols]
   [metabase.lib.schema.common :as lib.schema.common]
   [metabase.util :as u]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   #?@(:clj ([pretty.core :as pretty]))))

(defn- get-in-cache [cache ks]
  (when-some [cached-value (get-in @cache ks)]
    (when-not (= cached-value ::nil)
      cached-value)))

(defn- store-in-cache! [cache ks value]
  (let [value (if (some? value) value ::nil)]
    (swap! cache assoc-in ks value)
    (when-not (= value ::nil)
      value)))

(mu/defn ^:private store-database!
  [cache
   database-metadata :- lib.metadata/DatabaseMetadata]
  (let [database-metadata (-> database-metadata
                              (update-keys u/->kebab-case-en)
                              (assoc :lib/type :metadata/database))]
    (store-in-cache! cache [:metadata/database] database-metadata)))

(mu/defn ^:private store-metadata!
  [cache
   metadata-type :- [:enum :metadata/database :metadata/table :metadata/column :metadata/card :metadata/metric :metadata/segment]
   id            :- ::lib.schema.common/positive-int
   metadata      :- [:multi
                     {:dispatch :lib/type}
                     [:metadata/database lib.metadata/DatabaseMetadata]
                     [:metadata/table    lib.metadata/TableMetadata]
                     [:metadata/column   lib.metadata/ColumnMetadata]
                     [:metadata/card     lib.metadata/CardMetadata]
                     [:metadata/metric   lib.metadata/MetricMetadata]
                     [:metadata/segment  lib.metadata/SegmentMetadata]]]
  (let [metadata (-> metadata
                     (update-keys u/->kebab-case-en)
                     (assoc :lib/type metadata-type))]
    (store-in-cache! cache [metadata-type id] metadata)))

(defn- get-in-cache-or-fetch [cache ks fetch-thunk]
  (if-some [cached-value (get-in @cache ks)]
    (when-not (= cached-value ::nil)
      cached-value)
    (store-in-cache! cache ks (fetch-thunk))))

(defn- bulk-metadata [cache uncached-provider metadata-type ids]
  (when (seq ids)
    (log/debugf "Getting %s metadata with IDs %s" metadata-type (pr-str (sort ids)))
    (let [existing-ids (set (keys (get @cache metadata-type)))
          missing-ids  (set/difference (set ids) existing-ids)]
      (log/debugf "Already fetched %s: %s" metadata-type (pr-str (sort (set/intersection (set ids) existing-ids))))
      (when (seq missing-ids)
        (log/debugf "Need to fetch %s: %s" metadata-type (pr-str (sort missing-ids)))
        ;; TODO -- we should probably store `::nil` markers for things we tried to fetch that didn't exist
        (doseq [instance (lib.metadata.protocols/bulk-metadata uncached-provider metadata-type missing-ids)]
          (store-in-cache! cache [metadata-type (:id instance)] instance))))
    (for [id ids]
      (get-in-cache cache [metadata-type id]))))

(defn- tables [metadata-provider cache]
  (let [fetched-tables #(lib.metadata.protocols/tables metadata-provider)]
    (doseq [table fetched-tables]
      (store-in-cache! cache [:metadata/table (:id table)] table))
    fetched-tables))

(defn- fields [metadata-provider cache table-id]
  (let [fetched-fields (lib.metadata.protocols/fields metadata-provider table-id)]
    (doseq [field fetched-fields]
      (store-in-cache! cache [:metadata/column (:id field)] field))
    fetched-fields))

(defn- metrics [metadata-provider cache table-id]
  (let [fetched-metrics (lib.metadata.protocols/metrics metadata-provider table-id)]
    (doseq [metric fetched-metrics]
      (store-in-cache! cache [:metadata/metric (:id metric)] metric))
    fetched-metrics))

(deftype CachedProxyMetadataProvider [cache metadata-provider]
  lib.metadata.protocols/MetadataProvider
  (database [_this]            (get-in-cache-or-fetch cache [:metadata/database]            #(lib.metadata.protocols/database metadata-provider)))
  (table    [_this table-id]   (get-in-cache-or-fetch cache [:metadata/table table-id]      #(lib.metadata.protocols/table    metadata-provider table-id)))
  (field    [_this field-id]   (get-in-cache-or-fetch cache [:metadata/column field-id]     #(lib.metadata.protocols/field    metadata-provider field-id)))
  (card     [_this card-id]    (get-in-cache-or-fetch cache [:metadata/card card-id]        #(lib.metadata.protocols/card     metadata-provider card-id)))
  (metric   [_this metric-id]  (get-in-cache-or-fetch cache [:metadata/metric metric-id]    #(lib.metadata.protocols/metric   metadata-provider metric-id)))
  (segment  [_this segment-id] (get-in-cache-or-fetch cache [:metadata/segment segment-id]  #(lib.metadata.protocols/segment  metadata-provider segment-id)))
  (tables   [_this]            (get-in-cache-or-fetch cache [::database-tables]             #(tables metadata-provider cache)))
  (fields   [_this table-id]   (get-in-cache-or-fetch cache [::table-fields table-id]       #(fields metadata-provider cache table-id)))
  (metrics  [_this table-id]   (get-in-cache-or-fetch cache [::table-metrics table-id]      #(metrics metadata-provider cache table-id)))
  (setting  [_this setting]    (lib.metadata.protocols/setting metadata-provider setting))
  lib.metadata.protocols/CachedMetadataProvider
  (cached-database [_this]                           (get-in-cache    cache [:metadata/database]))
  (cached-metadata [_this metadata-type id]          (get-in-cache    cache [metadata-type id]))
  (store-database! [_this database-metadata]         (store-database! cache database-metadata))
  (store-metadata! [_this metadata-type id metadata] (store-metadata! cache metadata-type id metadata))
  ;; these only work if the underlying metadata provider is also a [[BulkMetadataProvider]].
  lib.metadata.protocols/BulkMetadataProvider
  (bulk-metadata [_this metadata-type ids]
    (bulk-metadata cache metadata-provider metadata-type ids))
  #?@(:clj
      [pretty/PrettyPrintable
       (pretty [_this]
               (list `cached-metadata-provider metadata-provider))]))

(defn cached-metadata-provider
  ^CachedProxyMetadataProvider [metadata-provider]
  (->CachedProxyMetadataProvider (atom {}) metadata-provider))

(ns metabase.lib.common
  (:require
   [metabase.lib.dispatch :as lib.dispatch]
   [metabase.lib.hierarchy :as lib.hierarchy]
   [metabase.lib.options :as lib.options]
   [metabase.lib.ref :as lib.ref]
   [metabase.lib.schema.common :as schema.common]
   [metabase.util.malli :as mu])
  #?(:cljs (:require-macros [metabase.lib.common])))

(comment lib.options/keep-me
         mu/keep-me)

(mu/defn external-op :- [:maybe ::schema.common/external-op]
  "Convert the internal operator `clause` to the external format."
  [[operator options :as clause]]
  (when clause
    {:lib/type :lib/external-op
     :operator (cond-> operator
                 (keyword? operator) name)
     :options  options
     :args     (subvec clause 2)}))

(defmulti ->op-arg
  {:arglists '([x])}
  lib.dispatch/dispatch-value
  :hierarchy lib.hierarchy/hierarchy)

(defmethod ->op-arg :default
  [x]
  (if (and (vector? x)
           (keyword? (first x)))
    ;; MBQL clause
    (mapv ->op-arg x)
    ;; Something else - just return it
    x))

(defmethod ->op-arg :dispatch-type/sequential
  [xs]
  (mapv ->op-arg xs))

(defmethod ->op-arg :metadata/column
  [field-metadata]
  (lib.ref/ref field-metadata))

(defmethod ->op-arg :metadata/metric
  [metric-def]
  (lib.ref/ref metric-def))

(defmethod ->op-arg :metadata/segment
  [segment-def]
  (lib.ref/ref segment-def))

(defmethod ->op-arg :lib/external-op
  [{:keys [operator options args] :or {options {}}}]
  (->op-arg (lib.options/ensure-uuid (into [(keyword operator) options]
                                           (map ->op-arg)
                                           args))))

(defn defop-create
  [op-name args]
  (into [op-name {:lib/uuid (str (random-uuid))}]
        (map ->op-arg)
        args))

#?(:clj
   (defmacro defop
     [op-name & argvecs]
     {:pre [(symbol? op-name)
            (every? vector? argvecs) (every? #(every? symbol? %) argvecs)
            (every? #(not-any? #{'query 'stage-number} %) argvecs)]}
     `(mu/defn ~op-name :- ~(keyword "mbql.clause" (name op-name))
        ~(format "Create a standalone clause of type `%s`." (name op-name))
        ~@(for [argvec argvecs
                :let [arglist-expr (if (contains? (set argvec) '&)
                                     (cons `list* (remove #{'&} argvec))
                                     argvec)]]
            `([~@argvec]
              (defop-create ~(keyword op-name) ~arglist-expr))))))

(ns metabase.shared.formatting.constants
  #?(:cljs (:require
            [metabase.shared.formatting.internal.date-builder :as builder])))

(defn abbreviated?
  [{:keys [output-density]}]
  (= output-density "compact"))

(defn condense-ranges?
  [{:keys [output-density]}]
  (#{"compact" "condensed"} output-density))

(def ^:export default-date-style
  "MMMM D, YYYY")

(def ^:export default-time-style
  "h:mm A")

(def ^:export known-date-styles
  {"M/D/YYYY"           [:month-d "/" :day-of-month-d "/" :year]
   "D/M/YYYY"           [:day-of-month-d "/" :month-d "/" :year]
   "YYYY/M/D"           [:year "/" :month-d "/" :day-of-month-d]
   "MMMM D, YYYY"       [:month-full " " :day-of-month-d ", " :year]
   "D MMMM, YYYY"       [:day-of-month-d " " :month-full ", " :year]
   "dddd, MMMM D, YYYY" [:day-of-week-full ", " :month-full " " :day-of-month-d ", " :year]})

(def ^:export known-time-styles
  {"h:mm A" [:hour-12-d  ":" :minute-dd " " :am-pm]
   "HH:mm"  [:hour-24-dd ":" :minute-dd]
   "HH"     [:hour-24-dd]})

(def ^:export known-datetime-styles
  {"M/D/YYYY, h:mm A" {:date-format (get known-date-styles "M/D/YYYY")
                       :time-format (get known-time-styles "h:mm A")}})

(ns metabase.lib.types.constants
  #?(:cljs (:require [goog.object :as gobj])))

#?(:cljs
   (do
     (def ^:export name->type
       "A map of Type name (as string, without `:type/` namespace) -> type keyword
         {\"Temporal\" :type/Temporal, ...}"
       (reduce (fn [m typ] (doto m (gobj/set (name typ) typ)))
               #js {}
               (distinct (mapcat descendants [:type/* :Semantic/* :Relation/*]))))
     ;; primary field types used for picking operators, etc
     (def ^:export key-number "JS-friendly access for the number type" ::number)
     (def ^:export key-string "JS-friendly access for the string type" ::string)
     (def ^:export key-string-like "JS-friendly access for the string-like type" ::string-like)
     (def ^:export key-boolean "JS-friendly access for the boolean type" ::boolean)
     (def ^:export key-temporal "JS-friendly access for the temporal type" ::temporal)
     (def ^:export key-location "JS-friendly access for the location type" ::location)
     (def ^:export key-coordinate "JS-friendly access for the coordinate type" ::coordinate)
     (def ^:export key-foreign-KEY "JS-friendly access for the foreign-key type" ::foreign-key)
     (def ^:export key-primary-KEY "JS-friendly access for the primary-key type" ::primary-key)
     (def ^:export key-json "JS-friendly access for the JSON type" ::json)
     (def ^:export key-xml "JS-friendly access for the JSON type" ::xml)
     (def ^:export key-structured "JS-friendly access for the structured type" ::structured)
     ;; other types used for various purposes
     (def ^:export key-summable "JS-friendly access for the summable type" ::summable)
     (def ^:export key-scope "JS-friendly access for the scope type" ::scope)
     (def ^:export key-category "JS-friendly access for the category type" ::category)
     (def ^:export key-unknown "JS-friendly access for the unknown type" ::unknown)))
;; NOTE: be sure not to create cycles using the "other" types
(def type-hierarchies
  {::temporal    {:effective-type [:type/Temporal]
                  :semantic-type  [:type/Temporal]}
   ::number      {:effective-type [:type/Number]
                  :semantic-type  [:type/Number]}
   ::integer     {:effective-type [:type/Integer]}
   ::string      {:effective-type [:type/Text]
                  :semantic-type  [:type/Text :type/Category]}
   ::string_like {:effective-type [:type/TextLike]}
   ::boolean     {:effective-type [:type/Boolean]}
   ::coordinate  {:semantic-type [:type/Coordinate]}
   ::location    {:semantic-type [:type/Address]}
   ::entity      {:semantic-type [:type/FK :type/PK :type/Name]}
   ::foreign_key {:semantic-type [:type/FK]}
   ::primary_key {:semantic-type [:type/PK]}
   ::json        {:effective-type [:type/SerializedJSON]}
   ::xml         {:effective-type [:type/XML]}
   ::structured  {:effective-type [:type/Structured]}
   ::summable    {:include [::number]
                  :exclude [::entity ::location ::temporal]}
   ::scope       {:include [::number ::temporal ::category ::entity ::string]
                  :exclude [::location]}
   ::category    {:effective-type [:type/Boolean]
                  :semantic-type  [:type/Category]
                  :include        [::location]}
   ;; NOTE: this is defunct right now.  see definition of metabase.lib.types.isa/dimension?.
   ::dimension   {:include [::temporal ::category ::entity]}})

(ns metabase.lib.convert
  (:require
   [clojure.data :as data]
   [clojure.set :as set]
   [clojure.string :as str]
   [malli.core :as mc]
   [malli.error :as me]
   [medley.core :as m]
   [metabase.lib.dispatch :as lib.dispatch]
   [metabase.lib.hierarchy :as lib.hierarchy]
   [metabase.lib.options :as lib.options]
   [metabase.lib.schema :as lib.schema]
   [metabase.lib.schema.expression :as lib.schema.expression]
   [metabase.lib.schema.ref :as lib.schema.ref]
   [metabase.lib.util :as lib.util]
   [metabase.mbql.normalize :as mbql.normalize]
   [metabase.util :as u]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu])
  #?@(:cljs [(:require-macros [metabase.lib.convert :refer [with-aggregation-list]])]))

(def ^:private ^:dynamic *pMBQL-uuid->legacy-index*
  {})

(def ^:private ^:dynamic *legacy-index->pMBQL-uuid*
  {})

(defn- clean-location [almost-stage error-type error-location]
  (let [operate-on-parent? #{:malli.core/missing-key :malli.core/end-of-input}
        location (if (operate-on-parent? error-type)
                   (drop-last 2 error-location)
                   (drop-last 1 error-location))
        [location-key] (if (operate-on-parent? error-type)
                         (take-last 2 error-location)
                         (take-last 1 error-location))]
    (if (seq location)
      (update-in almost-stage
                 location
                 (fn [error-loc]
                   (let [result (assoc error-loc location-key nil)]
                     (cond
                       (vector? error-loc) (into [] (remove nil?) result)
                       (map? error-loc) (u/remove-nils result)
                       :else result))))
      (dissoc almost-stage location-key))))

(def ^:private stage-keys
  #{:aggregation :breakout :expressions :fields :filters :order-by :joins})

(defn- clean-stage-schema-errors [almost-stage]
  (loop [almost-stage almost-stage
         removals []]
    (if-let [[error-type error-location] (->> (mc/explain ::lib.schema/stage.mbql almost-stage)
                                              :errors
                                              (filter (comp stage-keys first :in))
                                              (map (juxt :type :in))
                                              first)]
      (let [new-stage (clean-location almost-stage error-type error-location)]
        (log/warnf "Clean: Removing bad clause in %s due to error %s:\n%s"
                   (u/colorize :yellow (pr-str error-location))
                   (u/colorize :yellow (pr-str (or error-type
                                                   ;; if `error-type` is missing, which seems to happen sometimes,
                                                   ;; fall back to humanizing the entire error.
                                                   (me/humanize (mc/explain ::lib.schema/stage.mbql almost-stage)))))
                   (u/colorize :red (u/pprint-to-str (first (data/diff almost-stage new-stage)))))
        (if (= new-stage almost-stage)
          almost-stage
          (recur new-stage (conj removals [error-type error-location]))))
      almost-stage)))

(defn- clean-stage-ref-errors [almost-stage]
  (reduce (fn [almost-stage [loc _]]
              (clean-location almost-stage ::lib.schema/invalid-ref loc))
          almost-stage
          (lib.schema/ref-errors-for-stage almost-stage)))

(defn- clean-stage [almost-stage]
  (-> almost-stage
      clean-stage-schema-errors
      clean-stage-ref-errors))

(defn- clean [almost-query]
  (loop [almost-query almost-query
         stage-index 0]
    (let [current-stage (nth (:stages almost-query) stage-index)
          new-stage (clean-stage current-stage)]
      (if (= current-stage new-stage)
        (if (= stage-index (dec (count (:stages almost-query))))
          almost-query
          (recur almost-query (inc stage-index)))
        (recur (update almost-query :stages assoc stage-index new-stage) stage-index)))))

(defmulti ->pMBQL
  {:arglists '([x])}
  lib.dispatch/dispatch-value
  :hierarchy lib.hierarchy/hierarchy)

(defn- default-MBQL-clause->pMBQL [mbql-clause]
  (let [last-elem (peek mbql-clause)
        last-elem-option? (map? last-elem)
        [clause-type & args] (cond-> mbql-clause
                               last-elem-option? pop)
        options (if last-elem-option?
                  last-elem
                  {})]
    (lib.options/ensure-uuid (into [clause-type options] (map ->pMBQL) args))))

(defmethod ->pMBQL :default
  [x]
  (if (and (vector? x)
           (keyword? (first x)))
    (default-MBQL-clause->pMBQL x)
    x))

(defmethod ->pMBQL :mbql/query
  [query]
  query)

(def legacy-default-join-alias
  "__join")

(defn- deduplicate-join-aliases
  [joins]
  (let [unique-name-fn (lib.util/unique-name-generator)]
    (mapv (fn [join]
            (cond-> join
              (= (:alias join) legacy-default-join-alias) (update :alias unique-name-fn)))
          joins)))

(defn- stage-source-card-id->pMBQL
  [stage]
  (if (string? (:source-table stage))
    (-> stage
        (assoc :source-card (lib.util/legacy-string-table-id->card-id (:source-table stage)))
        (dissoc :source-table))
    stage))

#?(:clj
   (defmacro with-aggregation-list
     [aggregations & body]
     `(let [aggregations#  ~aggregations
            legacy->pMBQL# (into {}
                                 (map-indexed (fn [~'idx [~'_tag {~'ag-uuid :lib/uuid}]]
                                                [~'idx ~'ag-uuid]))
                                 aggregations#)
            pMBQL->legacy# (into {}
                                 (map-indexed (fn [~'idx [~'_tag {~'ag-uuid :lib/uuid}]]
                                                [~'ag-uuid ~'idx]))
                                 aggregations#)]
        (binding [*legacy-index->pMBQL-uuid* legacy->pMBQL#
                  *pMBQL-uuid->legacy-index* pMBQL->legacy#]
          ~@body))))

(defmethod ->pMBQL :mbql.stage/mbql
  [stage]
  (let [aggregations (->pMBQL (:aggregation stage))
        expressions  (->> stage
                          :expressions
                          (mapv (fn [[k v]]
                                  (-> v
                                      ->pMBQL
                                      (lib.util/top-level-expression-clause k))))
                          not-empty)]
    (metabase.lib.convert/with-aggregation-list aggregations
      (let [stage (-> stage
                      stage-source-card-id->pMBQL
                      (m/assoc-some :aggregation aggregations :expressions expressions))
            stage (reduce
                   (fn [stage k]
                     (if-not (get stage k)
                       stage
                       (update stage k ->pMBQL)))
                   stage
                   (disj stage-keys :aggregation :expressions))]
        (cond-> stage
          (:joins stage) (update :joins deduplicate-join-aliases))))))

(defmethod ->pMBQL :mbql.stage/native
  [stage]
  (m/update-existing stage :template-tags update-vals (fn [tag] (m/update-existing tag :dimension ->pMBQL))))

(defmethod ->pMBQL :mbql/join
  [join]
  (let [join (-> join
                 (update :conditions ->pMBQL)
                 (update :stages ->pMBQL))]
    (cond-> join
      (:fields join) (update :fields (fn [fields]
                                       (if (seqable? fields)
                                         (mapv ->pMBQL fields)
                                         (keyword fields))))
      (not (:alias join)) (assoc :alias legacy-default-join-alias))))

(defmethod ->pMBQL :dispatch-type/sequential
  [xs]
  (mapv ->pMBQL xs))

(defmethod ->pMBQL :dispatch-type/map
  [m]
  (if (:type m)
    (-> (lib.util/pipeline m)
        (update :stages (fn [stages]
                          (mapv ->pMBQL stages)))
        (assoc :lib.convert/converted? true)
        clean)
    (update-vals m ->pMBQL)))

(defmethod ->pMBQL :field
  [[_tag x y]]
  (let [[id-or-name options] (if (map? x)
                               [y x]
                               [x y])]
    (lib.options/ensure-uuid [:field options id-or-name])))

(defmethod ->pMBQL :value
  [[_tag value opts]]
  ;; `:value` uses `:snake_case` keys in legacy MBQL for some insane reason (actually this was to match the shape of
  ;; the keys in Field metadata), at least for the three type keys enumerated below.
  ;; See [[metabase.mbql.schema/ValueTypeInfo]].
  (let [opts (set/rename-keys opts {:base_type     :base-type
                                    :semantic_type :semantic-type
                                    :database_type :database-type})
        ;; in pMBQL, `:effective-type` is a required key for `:value`. `:value` SHOULD have always had `:base-type`,
        ;; but on the off chance it did not, get the type from value so the schema doesn't fail entirely.
        opts (assoc opts :effective-type (or (:effective-type opts)
                                             (:base-type opts)
                                             (lib.schema.expression/type-of value)))]
    (lib.options/ensure-uuid [:value opts value])))

(defmethod ->pMBQL :case
  [[_tag pred-expr-pairs options]]
  (let [default (:default options)]
    (cond-> [:case (dissoc options :default) (mapv ->pMBQL pred-expr-pairs)]
      :always lib.options/ensure-uuid
      (some? default) (conj (->pMBQL default)))))

(defmethod ->pMBQL :expression
  [[tag value opts]]
  (lib.options/ensure-uuid [tag opts value]))

(defn- get-or-throw!
  [m k]
  (let [result (get m k ::not-found)]
    (if-not (= result ::not-found)
      result
      (throw (ex-info (str "Unable to find key " (pr-str k) " in map.")
                      {:m m
                       :k k})))))

(defmethod ->pMBQL :aggregation
  [[tag aggregation-index opts, :as clause]]
  (lib.options/ensure-uuid
   [tag opts (or (get *legacy-index->pMBQL-uuid* aggregation-index)
                 (throw (ex-info (str "Error converting :aggregation reference: no aggregation at index "
                                      aggregation-index)
                                 {:clause clause})))]))

(defmethod ->pMBQL :aggregation-options
  [[_tag aggregation options]]
  (let [[tag opts & args] (->pMBQL aggregation)]
    (into [tag (merge opts options)] args)))

(defn legacy-query-from-inner-query
  [database-id inner-query]
  (merge {:database database-id, :type :query}
         (if (:native inner-query)
           {:native (set/rename-keys inner-query {:native :query})}
           {:query inner-query})))

(defmulti ->legacy-MBQL
  {:arglists '([x])}
  lib.dispatch/dispatch-value
  :hierarchy lib.hierarchy/hierarchy)

(defn- metabase-lib-keyword?
  [k]
  (and (qualified-keyword? k)
       (when-let [symb-namespace (namespace k)]
         (or (= symb-namespace "lib")
             (str/starts-with? symb-namespace "metabase.lib.")))))

(defn- disqualify
  ([]
   (remove (fn [[k _v]]
             (metabase-lib-keyword? k))))
  ([m]
   (into {} (disqualify) m)))

(defn- options->legacy-MBQL
  [m]
  (not-empty
   (into {}
         (comp (disqualify)
               (remove (fn [[k _v]]
                         (= k :effective-type))))
         m)))

(defn- aggregation->legacy-MBQL [[tag options & args]]
  (let [inner (into [tag] (map ->legacy-MBQL) args)
        ;; the default value of the :case expression is in the options
        ;; in legacy MBQL
        inner (if (and (= tag :case) (next args))
                (conj (pop inner) {:default (peek inner)})
                inner)]
    (if-let [aggregation-opts (not-empty (options->legacy-MBQL options))]
      [:aggregation-options inner aggregation-opts]
      inner)))

(defn- clause-with-options->legacy-MBQL [[k options & args]]
  (if (map? options)
    (into [k] (concat (map ->legacy-MBQL args)
                      (when-let [options (options->legacy-MBQL options)]
                        [options])))
    (into [k] (map ->legacy-MBQL (cons options args)))))

(defmethod ->legacy-MBQL :default
  [x]
  (cond
    (and (vector? x)
         (keyword? (first x))) (clause-with-options->legacy-MBQL x)
    (map? x)                   (-> x
                                   disqualify
                                   (update-vals ->legacy-MBQL))
    :else x))

(doseq [tag [::aggregation ::expression]]
  (lib.hierarchy/derive tag ::aggregation-or-expression))

(doseq [tag [:count :avg :count-where :distinct
             :max :median :min :percentile
             :share :stddev :sum :sum-where]]
  (lib.hierarchy/derive tag ::aggregation))

(doseq [tag [:+ :- :* :/
             :case :coalesce
             :abs :log :exp :sqrt :ceil :floor :round :power :interval
             :relative-datetime :time :absolute-datetime :now :convert-timezone
             :get-week :get-year :get-month :get-day :get-hour
             :get-minute :get-second :get-quarter
             :datetime-add :datetime-subtract
             :concat :substring :replace :regexextract :regex-match-first
             :length :trim :ltrim :rtrim :upper :lower]]
  (lib.hierarchy/derive tag ::expression))

(defmethod ->legacy-MBQL ::aggregation-or-expression
  [input]
  (aggregation->legacy-MBQL input))

(defn- stage-metadata->legacy-metadata [stage-metadata]
  (into []
        (comp (map #(update-keys % u/->snake_case_en))
              (map ->legacy-MBQL))
        (:columns stage-metadata)))

(defn- chain-stages [{:keys [stages]}]
  ;; :source-metadata aka :lib/stage-metadata is handled differently in the two formats.
  ;; In legacy, an inner query might have both :source-query, and :source-metadata giving the metadata for that nested
  ;; :source-query.
  ;; In pMBQL, the :lib/stage-metadata is attached to the same stage it applies to.
  ;; So when chaining pMBQL stages back into legacy form, if stage n has :lib/stage-metadata, stage n+1 needs
  ;; :source-metadata attached.
  (let [inner-query (first (reduce (fn [[inner stage-metadata] stage]
                                     [(cond-> (->legacy-MBQL stage)
                                        inner          (assoc :source-query inner)
                                        stage-metadata (assoc :source-metadata (stage-metadata->legacy-metadata stage-metadata)))
                                      ;; Get the :lib/stage-metadata off the original pMBQL stage, not the converted one.
                                      (:lib/stage-metadata stage)])
                                   nil
                                   stages))]
    (cond-> inner-query
      ;; If this is a native query, inner query will be used like: `{:type :native :native #_inner-query {:query ...}}`
      (:native inner-query) (set/rename-keys {:native :query}))))

(defmethod ->legacy-MBQL :dispatch-type/map [m]
  (into {}
        (comp (disqualify)
              (map (fn [[k v]]
                     [k (->legacy-MBQL v)])))
        m))

(defmethod ->legacy-MBQL :aggregation [[_ opts agg-uuid :as ag]]
  (if (map? opts)
    (try
      (let [opts (options->legacy-MBQL opts)]
        (cond-> [:aggregation (get-or-throw! *pMBQL-uuid->legacy-index* agg-uuid)]
          opts (conj opts)))
      (catch #?(:clj Throwable :cljs :default) e
        (throw (ex-info (lib.util/format "Error converting aggregation reference to pMBQL: %s" (ex-message e))
                        {:ref ag}
                        e))))
    ;; Our conversion is a bit too aggressive and we're hitting legacy refs like [:aggregation 0] inside source_metadata that are only used for legacy and thus can be ignored
    ag))

(defmethod ->legacy-MBQL :dispatch-type/sequential [xs]
  (mapv ->legacy-MBQL xs))

(defmethod ->legacy-MBQL :field [[_ opts id]]
  ;; Fields are not like the normal clauses - they need that options field even if it's null.
  ;; TODO: Sometimes the given field is in the legacy order - that seems wrong.
  (let [[opts id] (if (or (nil? opts) (map? opts))
                    [opts id]
                    [id opts])]
    [:field
     (->legacy-MBQL id)
     (options->legacy-MBQL opts)]))

(defmethod ->legacy-MBQL :value
  [[_tag opts value]]
  (let [opts (-> opts
                 ;; as mentioned above, `:value` in legacy MBQL expects `snake_case` keys for type info keys.
                 (set/rename-keys  {:base-type     :base_type
                                    :semantic-type :semantic_type
                                    :database-type :database_type})
                 options->legacy-MBQL)]
    ;; in legacy MBQL, `:value` has to be three args; `opts` has to be present, but it should can be `nil` if it is
    ;; empty.
    [:value value opts]))

(defn- update-list->legacy-boolean-expression
  [m pMBQL-key legacy-key]
  (cond-> m
    (= (count (get m pMBQL-key)) 1) (m/update-existing pMBQL-key (comp ->legacy-MBQL first))
    (> (count (get m pMBQL-key)) 1) (m/update-existing pMBQL-key #(into [:and] (map ->legacy-MBQL) %))
    :always (set/rename-keys {pMBQL-key legacy-key})))

(defmethod ->legacy-MBQL :mbql/join [join]
  (let [base (cond-> (disqualify join)
               (str/starts-with? (:alias join) legacy-default-join-alias) (dissoc :alias))]
    (merge (-> base
               (dissoc :stages :conditions)
               (update-vals ->legacy-MBQL))
           (-> base
               (select-keys [:conditions])
               (update-list->legacy-boolean-expression :conditions :condition))
           (chain-stages base))))

(defn- source-card->legacy-source-table
  [stage]
  (if-let [source-card-id (:source-card stage)]
    (-> stage
        (dissoc :source-card)
        (assoc :source-table (str "card__" source-card-id)))
    stage))

(defmethod ->legacy-MBQL :mbql.stage/mbql
  [stage]
  (metabase.lib.convert/with-aggregation-list (:aggregation stage)
    (reduce #(m/update-existing %1 %2 ->legacy-MBQL)
            (-> stage
                disqualify
                source-card->legacy-source-table
                (m/update-existing :aggregation #(mapv aggregation->legacy-MBQL %))
                (m/update-existing :expressions (fn [expressions]
                                                  (into {}
                                                        (for [expression expressions
                                                              :let [legacy-clause (->legacy-MBQL expression)]]
                                                          [(lib.util/expression-name expression)
                                                           ;; We wrap literals in :value ->pMBQL
                                                           ;; so unwrap this direction
                                                           (if (= :value (first legacy-clause))
                                                             (second legacy-clause)
                                                             legacy-clause)]))))
                (update-list->legacy-boolean-expression :filters :filter))
            (disj stage-keys :aggregation :filters :expressions))))

(defmethod ->legacy-MBQL :mbql.stage/native [stage]
  (-> stage
      disqualify
      (update-vals ->legacy-MBQL)))

(defmethod ->legacy-MBQL :mbql/query [query]
  (try
    (let [base        (disqualify query)
          parameters  (:parameters base)
          inner-query (chain-stages base)
          query-type  (if (-> query :stages last :lib/type (= :mbql.stage/native))
                        :native
                        :query)]
      (merge (-> base
                 (dissoc :stages :parameters :lib.convert/converted?)
                 (update-vals ->legacy-MBQL))
             (cond-> {:type query-type query-type inner-query}
               (seq parameters) (assoc :parameters parameters))))
    (catch #?(:clj Throwable :cljs :default) e
      (throw (ex-info (lib.util/format "Error converting MLv2 query to legacy query: %s" (ex-message e))
                      {:query query}
                      e)))))

(mu/defn legacy-ref->pMBQL :- ::lib.schema.ref/ref
  "Convert a legacy MBQL `:field`/`:aggregation`/`:expression` reference to pMBQL. Normalizes the reference if needed,
  and handles JS -> Clj conversion as needed."
  ([query legacy-ref]
   (legacy-ref->pMBQL query -1 legacy-ref))
  ([query        :- ::lib.schema/query
    stage-number :- :int
    legacy-ref   :- some?]
   (let [legacy-ref                  (->> #?(:clj legacy-ref :cljs (js->clj legacy-ref :keywordize-keys true))
                                          (mbql.normalize/normalize-fragment nil))
         {aggregations :aggregation} (lib.util/query-stage query stage-number)]
     (with-aggregation-list aggregations
       (try
         (->pMBQL legacy-ref)
         (catch #?(:clj Throwable :cljs :default) e
           (throw (ex-info (lib.util/format "Error converting legacy ref to pMBQL: %s" (ex-message e))
                           {:query                    query
                            :stage-number             stage-number
                            :legacy-ref               legacy-ref
                            :legacy-index->pMBQL-uuid *legacy-index->pMBQL-uuid*}
                           e))))))))

(defn- from-json [query-fragment]
  #?(:cljs (if (object? query-fragment)
             (js->clj query-fragment :keywordize-keys true)
             query-fragment)
     :clj  query-fragment))

(defn js-legacy-query->pMBQL
  [query-map]
  (-> query-map
      from-json
      (u/assoc-default :type :query)
      mbql.normalize/normalize
      ->pMBQL))

(defn js-legacy-inner-query->pMBQL
  [inner-query]
  (js-legacy-query->pMBQL {:type  :query
                           :query (from-json inner-query)}))

(ns metabase.shared.formatting.internal.date-builder
  (:require
   [clojure.string :as str])
  #?(:clj (:import
           java.time.format.DateTimeFormatter)))

(def format-strings
  {:year              #?(:cljs "YYYY" :clj "yyyy")  ; 2022
   :quarter           "Q"                           ; 2 ("Q2" etc. is added by higher level formatting)
   :month-full        "MMMM"                        ; April
   :month-short       "MMM"                         ; Apr
   :month-dd          "MM"                          ; 04
   :month-d           "M"                           ; 4
   :day-of-month-d    #?(:cljs "D"    :clj "d")     ; 6
   :day-of-month-dd   #?(:cljs "DD"   :clj "dd")    ; 06
   :day-of-week-full  #?(:cljs "dddd" :clj "EEEE")  ; Friday
   :day-of-week-short #?(:cljs "ddd"  :clj "EEE")   ; Fri
   :hour-24-dd        "HH"                          ; 17, 05
   :hour-24-d         "H"                           ; 17, 5
   :hour-12-dd        "hh"                          ; 05
   :hour-12-d         "h"                           ; 5
   :am-pm             #?(:cljs "A"    :clj "a")     ; AM
   :minute-d          "m"                           ; 7, 39
   :minute-dd         "mm"                          ; 07, 39
   :second-dd         "ss"                          ; 08, 45
   :millisecond-ddd   "SSS"                         ; 001, 423
   :day-of-year       #?(:cljs "DDD"  :clj "D")     ; 235
   :week-of-year      #?(:cljs "wo"   :clj "w")})   ; 34th in CLJS, 34 in CLJ. No ordinal numbers in Java.

(defn- format-string-literal [lit]
  #?(:cljs (str "[" lit "]")
     :clj  (str "'" (str/replace lit "'" "''") "'")))

(defn ->formatter
  [format-list]
  (let [js->clj   #?(:cljs js->clj :clj identity)
        parts     (for [fmt (js->clj format-list)]
                    (cond
                      (keyword? fmt)             (get format-strings fmt)
                      (= fmt ":")                (format-string-literal ":")
                      (str/starts-with? fmt ":") (-> fmt (subs 1) keyword format-strings)
                      (string? fmt)              (format-string-literal fmt)
                      :else                      (throw (ex-info "Unknown element of date format"
                                                                 {:bad-element fmt
                                                                  :format      format-list}))))
        fmt-str   (apply str parts)]
    #?(:cljs #(.format % fmt-str)
       :clj  (let [formatter (DateTimeFormatter/ofPattern fmt-str)]
               #(.format formatter %)))))

(ns metabase.util.devtools
  ;; This special context is defined only for dev-mode shadow-cljs builds; see shadow-cljs.edn
  ;; In release builds, and JVM Clojure, this file is an empty namespace.
  #?(:cljs-dev (:require
                 [devtools.core :as devtools]
                 [shadow.cljs.devtools.client.browser])))

(ns metabase.lib.equality
  (:refer-clojure :exclude [=])
  (:require
   [medley.core :as m]
   [metabase.lib.card :as lib.card]
   [metabase.lib.convert :as lib.convert]
   [metabase.lib.dispatch :as lib.dispatch]
   [metabase.lib.hierarchy :as lib.hierarchy]
   [metabase.lib.metadata :as lib.metadata]
   [metabase.lib.options :as lib.options]
   [metabase.lib.ref :as lib.ref]
   [metabase.lib.schema :as lib.schema]
   [metabase.lib.schema.id :as lib.schema.id]
   [metabase.lib.schema.metadata :as lib.schema.metadata]
   [metabase.lib.schema.ref :as lib.schema.ref]
   [metabase.lib.util :as lib.util]
   [metabase.util.malli :as mu]
   #?@(:clj ([metabase.util.log :as log]))))

(defmulti =
  {:arglists '([x y])}
  ;; two things with different dispatch values (for maps, the `:lib/type` key; for MBQL clauses, the tag, and for
  ;; everything else, the `:dispatch-type/*` key) can't be equal.
  (fn [x y]
    (let [x-dispatch-value (lib.dispatch/dispatch-value x)
          y-dispatch-value (lib.dispatch/dispatch-value y)]
      (if (not= x-dispatch-value y-dispatch-value)
        ::different-dispatch-values
        x-dispatch-value)))
  :hierarchy lib.hierarchy/hierarchy)

(defmethod = ::different-dispatch-values
  [_x _y]
  false)

(defn- relevant-keys-set
  [m]
  (into #{}
        (remove (fn [k]
                  (and (qualified-keyword? k)
                       (not= k :lib/type))))
        (keys m)))

(defmethod = :dispatch-type/map
  [m1 m2]
  (let [m1-keys (relevant-keys-set m1)
        m2-keys (relevant-keys-set m2)]
    (and (clojure.core/= m1-keys m2-keys)
         (every? (fn [k]
                   (= (get m1 k)
                      (get m2 k)))
                 m1-keys))))

(defmethod = :dispatch-type/sequential
  [xs ys]
  (and (clojure.core/= (count xs) (count ys))
       (loop [[x & more-x] xs, [y & more-y] ys]
         (and (= x y)
              (or (empty? more-x)
                  (recur more-x more-y))))))

(def ^:private ^:dynamic *side->uuid->index* nil)

(defn- aggregation-uuid->index
  [stage]
  (into {}
        (map-indexed (fn [idx [_tag {ag-uuid :lib/uuid}]]
                       [ag-uuid idx]))
        (:aggregation stage)))

(defmethod = :mbql.stage/mbql
  [x y]
  (binding [*side->uuid->index* {:left (aggregation-uuid->index x)
                                 :right (aggregation-uuid->index y)}]
    ((get-method = :dispatch-type/map) x y)))

(defmethod = :aggregation
  [[x-tag x-opts x-uuid :as x] [y-tag y-opts y-uuid :as y]]
  (and (clojure.core/= 3 (count x) (count y))
       (clojure.core/= x-tag y-tag)
       (= x-opts y-opts)
       ;; If nil, it means we aren't comparing a stage, so just compare the uuid directly
       (if *side->uuid->index*
         (clojure.core/= (get-in *side->uuid->index* [:left x-uuid] ::no-left)
                         (get-in *side->uuid->index* [:right y-uuid] ::no-right))
         (clojure.core/= x-uuid y-uuid))))

(defmethod = :default
  [x y]
  (cond
    (map? x)        ((get-method = :dispatch-type/map) x y)
    (sequential? x) ((get-method = :dispatch-type/sequential) x y)
    :else           (clojure.core/= x y)))

(mu/defn resolve-field-id :- ::lib.schema.metadata/column
  "Integer Field ID: get metadata from the metadata provider. If this is the first stage of the query, merge in
  Saved Question metadata if available.
  This doesn't really have a good home. It's used here and by [[metabase.lib.field]], but because it depends on eg.
  [[metabase.lib.card]] and [[metabase.lib.convert]] it can't go in [[metabase.lib.metadata.calculation]]."
  [query        :- ::lib.schema/query
   stage-number :- :int
   field-id     :- ::lib.schema.id/field]
  (merge
   (when (lib.util/first-stage? query stage-number)
     (when-let [card-id (lib.util/source-card-id query)]
       (when-let [card-metadata (lib.card/saved-question-metadata query card-id)]
         (m/find-first #(clojure.core/= (:id %) field-id)
                       card-metadata))))
   (try
     (lib.metadata/field query field-id)
     (catch #?(:clj Throwable :cljs :default) _
       nil))))

(mu/defn ^:private column-join-alias :- [:maybe :string]
  [column :- ::lib.schema.metadata/column]
  ((some-fn :metabase.lib.join/join-alias :source-alias) column))

(mu/defn ^:private matching-join? :- :boolean
  [[_ref-kind {:keys [join-alias source-field]} _ref-id] :- ::lib.schema.ref/ref
   column                                                :- ::lib.schema.metadata/column]
  ;; If the ref has a source-field, and it matches the column's :fk-field-id then this is an implicitly joined field.
  ;; Implicitly joined columns have :source-alias ("PRODUCTS__via__PRODUCT_ID") but the refs don't have any join alias.
  (or (and source-field
           (clojure.core/= source-field (:fk-field-id column)))
      ;; If it's not an implicit join, then either the join aliases must match for an explicit join, or both be nil for
      ;; an own column.
      (clojure.core/= (column-join-alias column) join-alias)))

(mu/defn ^:private plausible-matches-for-name :- [:sequential ::lib.schema.metadata/column]
  [[_ref-kind _opts ref-name :as a-ref] :- ::lib.schema.ref/ref
   columns                              :- [:sequential ::lib.schema.metadata/column]]
  (or (not-empty (filter #(and (clojure.core/= (:lib/desired-column-alias %) ref-name)
                               (matching-join? a-ref %))
                         columns))
      (filter #(and (clojure.core/= (:name %) ref-name)
                    (matching-join? a-ref %))
              columns)))

(mu/defn ^:private plausible-matches-for-id :- [:sequential ::lib.schema.metadata/column]
  [[_ref-kind opts ref-id :as a-ref] :- ::lib.schema.ref/ref
   columns                           :- [:sequential ::lib.schema.metadata/column]
   generous?                         :- [:maybe :boolean]]
  (or (not-empty (filter #(and (clojure.core/= (:id %) ref-id)
                               ;; TODO: If the target ref has no join-alias, AND the source is fields or card, the join
                               ;; alias on the column can be ignored. QP can set it when it shouldn't. See #33972.
                               (or (and (not (:join-alias opts))
                                        (#{:source/fields :source/card} (:lib/source %)))
                                   (matching-join? a-ref %)))
                         columns))
      (when generous?
        (not-empty (filter #(clojure.core/= (:id %) ref-id) columns)))
      []))

(defn- ambiguous-match-error [a-ref columns]
  (ex-info "Ambiguous match! Implement more logic in disambiguate-matches."
           {:ref a-ref
            :columns columns}))

(mu/defn ^:private expression-column? [column]
  (or (= (:lib/source column) :source/expressions)
      (:lib/expression-name column)))

(mu/defn ^:private disambiguate-matches-dislike-field-refs-to-expressions :- [:maybe ::lib.schema.metadata/column]
  "If a custom column is a simple wrapper for a field, that column gets `:id`, `:table_id`, etc.
  A custom column should get a ref like `[:expression {} \"expr name\"]`, not `[:field {} 17]`.
  If we got a `:field` ref, prefer matches which are not `:lib/source :source/expressions`."
  [a-ref   :- ::lib.schema.ref/ref
   columns :- [:sequential ::lib.schema.metadata/column]]
  (or (when (= (first a-ref) :field)
        (when-let [non-exprs (not-empty (remove expression-column? columns))]
          (when-not (next non-exprs)
            (first non-exprs))))
      ;; In all other cases, this is an ambiguous match.
      #_(throw (ambiguous-match-error a-ref columns))
      #?(:cljs (js/console.warn (ambiguous-match-error a-ref columns))
         :clj  (log/warn (ambiguous-match-error a-ref columns)))))

(mu/defn ^:private disambiguate-matches-prefer-explicit :- [:maybe ::lib.schema.metadata/column]
  "Prefers table-default or explicitly joined columns over implicitly joinable ones."
  [a-ref   :- ::lib.schema.ref/ref
   columns :- [:sequential ::lib.schema.metadata/column]]
  (if-let [no-implicit (not-empty (remove :fk-field-id columns))]
    (if-not (next no-implicit)
      (first no-implicit)
      (disambiguate-matches-dislike-field-refs-to-expressions a-ref no-implicit))
    nil))

(mu/defn ^:private disambiguate-matches-no-alias :- [:maybe ::lib.schema.metadata/column]
  [a-ref   :- ::lib.schema.ref/ref
   columns :- [:sequential ::lib.schema.metadata/column]]
  ;; a-ref without :join-alias - if exactly one column has no :source-alias, that's the match.
  ;; ignore the source alias on columns with :source/card or :source/fields
  (if-let [no-alias (not-empty (remove #(and (column-join-alias %)
                                             (not (#{:source/card} (:lib/source %))))
                                       columns))]
    ;; At least 1 matching column with no :source-alias.
    (if-not (next no-alias)
      (first no-alias)
      ;; More than 1, keep digging.
      (disambiguate-matches-prefer-explicit a-ref no-alias))
    ;; No columns are missing :source-alias - pass them all to the next stage.
    ;; TODO: I'm not certain this one is sound, but it's necessary to make `lib.join/select-home-column` work as
    ;; written. If this case causes issues, that logic may need rewriting.
    nil))

(mu/defn ^:private disambiguate-matches :- [:maybe ::lib.schema.metadata/column]
  [a-ref   :- ::lib.schema.ref/ref
   columns :- [:sequential ::lib.schema.metadata/column]]
  (let [{:keys [join-alias]} (lib.options/options a-ref)]
    (if join-alias
      ;; a-ref has a :join-alias, match on that. Return nil if nothing matches.
      (when-let [matches (not-empty (filter #(clojure.core/= (column-join-alias %) join-alias) columns))]
        (if-not (next matches)
          (first matches)
          (#?(:cljs js/console.warn :clj log/warn)
           "Multiple plausible matches with the same :join-alias - more disambiguation needed"
           {:ref     a-ref
            :matches matches})
          #_(throw (ex-info "Multiple plausible matches with the same :join-alias - more disambiguation needed"
                          {:ref     a-ref
                           :matches matches}))))
      (disambiguate-matches-no-alias a-ref columns))))

(def ^:private FindMatchingColumnOptions
  [:map [:generous? {:optional true} :boolean]])

(mu/defn find-matching-column :- [:maybe ::lib.schema.metadata/column]
  "Given `a-ref-or-column` and a list of `columns`, finds the column that best matches this ref or column.
  Matching is based on finding the basically plausible matches first. There is often zero or one plausible matches, and
  this can return quickly.
  If there are multiple plausible matches, they are disambiguated by the most important extra included in the `ref`.
  (`:join-alias` first, then `:temporal-unit`, etc.)
  - Integer IDs in the `ref` are matched by ID; this usually is unambiguous.
    - If there are multiple joins on one table (including possible implicit joins), check `:join-alias` next.
      - If `a-ref` has a `:join-alias`, only a column which matches it can be the match, and it should be unique.
      - If `a-ref` doesn't have a `:join-alias`, prefer the column with no `:join-alias`, and prefer already selected
        columns over implicitly joinable ones.
    - There may be broken cases where the ref has an ID but the column does not. Therefore the ID must be resolved to a
      name or `:lib/desired-column-alias` and matched that way.
      - `query` and `stage-number` are required for this case, since they're needed to resolve the correct name.
      - Columns with `:id` set are dropped to prevent them matching. (If they didn't match by `:id` above they shouldn't
        match by name due to a coincidence of column names in different tables.)
  - String IDs are checked against `:lib/desired-column-alias` first.
    - If that doesn't match any columns, `:name` is compared next.
    - The same disambiguation (by `:join-alias` etc.) is applied if there are multiple plausible matches.
  Returns the matching column, or nil if no match is found."
  ([a-ref columns]
   (find-matching-column a-ref columns {}))
  ([[ref-kind _opts ref-id :as a-ref] :- ::lib.schema.ref/ref
    columns                           :- [:sequential ::lib.schema.metadata/column]
    {:keys [generous?]}               :- FindMatchingColumnOptions]
   (case ref-kind
     ;; Aggregations are referenced by the UUID of the column being aggregated.
     :aggregation  (m/find-first #(and (clojure.core/= (:lib/source %) :source/aggregations)
                                       (clojure.core/= (:lib/source-uuid %) ref-id))
                                 columns)
     ;; Expressions are referenced by name; fields by ID or name.
     (:expression
       :field)     (let [plausible (if (string? ref-id)
                                     (plausible-matches-for-name a-ref columns)
                                     (plausible-matches-for-id   a-ref columns generous?))]
                     (case (count plausible)
                       0 nil
                       1 (first plausible)
                       (disambiguate-matches a-ref plausible)))
     (throw (ex-info "Unknown type of ref" {:ref a-ref}))))
  ([query stage-number a-ref-or-column columns]
   (find-matching-column query stage-number a-ref-or-column columns {}))
  ([query           :- [:maybe ::lib.schema/query]
    stage-number    :- :int
    a-ref-or-column :- [:or ::lib.schema.metadata/column ::lib.schema.ref/ref]
    columns         :- [:sequential ::lib.schema.metadata/column]
    opts            :- FindMatchingColumnOptions]
   (let [[_ref-kind _opts ref-id :as a-ref] (if (lib.util/clause? a-ref-or-column)
                                              a-ref-or-column
                                              (lib.ref/ref a-ref-or-column))]
     (or (find-matching-column a-ref columns opts)
         ;; We failed to match by ID, so try again with the column's name. Any columns with `:id` set are dropped.
         ;; Why? Suppose there are two CREATED_AT columns in play - if one has an :id and it failed to match above, then
         ;; it certainly shouldn't match by name just because of the coincidence of column names!
       (when (and query (number? ref-id))
         (when-let [no-id-columns (not-empty (remove :id columns))]
           (when-let [resolved (if (lib.util/clause? a-ref-or-column)
                                 (resolve-field-id query stage-number ref-id)
                                 a-ref-or-column)]
             (find-matching-column (-> (assoc a-ref 2 (or (:lib/desired-column-alias resolved)
                                                          (:name resolved)))
                                       ;; make sure the :field ref has a `:base-type`, it's against the rules for a
                                       ;; nominal :field ref not to have a base-type -- this can fail schema
                                       ;; validation if it's missing in the Field ID ref we generate the nominal ref
                                       ;; from.
                                       (lib.options/update-options (partial merge {:base-type :type/*})))
                                   no-id-columns
                                   opts))))))))

(defn- ref-id-or-name [[_ref-kind _opts id-or-name]]
  id-or-name)

(mu/defn find-matching-ref :- [:maybe ::lib.schema.ref/ref]
  "Given `column` and a list of `refs`, finds the ref that best matches this column.
  Throws if there are multiple, ambiguous matches.
  Returns the matching ref, or nil if no plausible matches are found."
  [column :- ::lib.schema.metadata/column
   refs   :- [:sequential ::lib.schema.ref/ref]]
  (let [ref-tails (group-by ref-id-or-name refs)
        matches   (or (some->> column :lib/source-uuid (get ref-tails) not-empty)
                      (not-empty (get ref-tails (:id column)))
                      (not-empty (get ref-tails (:lib/desired-column-alias column)))
                      (get ref-tails (:name column))
                      [])]
    (case (count matches)
      0 nil
      1 (first matches)
      (throw (ex-info "Ambiguous match: given column matches multiple refs"
                      {:column        column
                       :matching-refs matches})))))

(mu/defn find-column-indexes-for-refs :- [:sequential :int]
  "Given a list `haystack` of columns or refs, and a list `needles` of refs to searc for, this returns a list parallel
  to `needles` with the corresponding index into the `haystack`, or -1 if not found.
  DISCOURAGED: This is intended for use only by [[metabase.lib.js/find-column-indexes-from-legacy-refs]].
  Other MLv2 code should use [[find-matching-column]] if the `haystack` is columns, or
  [[find-matching-ref]] if it's refs."
  [query        :- ::lib.schema/query
   stage-number :- :int
   needles      :- [:sequential ::lib.schema.ref/ref]
   haystack     :- [:sequential ::lib.schema.metadata/column]]
  (let [by-column (into {}
                        (map-indexed (fn [index column]
                                       [column index]))
                        haystack)]
    (for [needle needles
          :let [matched (find-matching-column query stage-number needle haystack)]]
      (get by-column matched -1))))

(mu/defn find-column-for-legacy-ref :- [:maybe ::lib.schema.metadata/column]
  "Like [[find-matching-column]], but takes a legacy MBQL reference. The name here is for consistency with other
  FE names for similar functions."
  ([query legacy-ref metadatas]
   (find-column-for-legacy-ref query -1 legacy-ref metadatas))
  ([query        :- ::lib.schema/query
    stage-number :- :int
    legacy-ref   :- :some
    metadatas    :- [:maybe [:sequential ::lib.schema.metadata/column]]]
   (find-matching-column query stage-number (lib.convert/legacy-ref->pMBQL query stage-number legacy-ref) metadatas)))

(defn mark-selected-columns
  ([cols selected-columns-or-refs]
   (mark-selected-columns nil -1 cols selected-columns-or-refs))
  ([query stage-number cols selected-columns-or-refs]
   (when (seq cols)
     (let [selected-refs          (mapv lib.ref/ref selected-columns-or-refs)
           matching-selected-cols (into #{}
                                        (map #(find-matching-column query stage-number % cols))
                                        selected-refs)]
       (mapv #(assoc % :selected? (contains? matching-selected-cols %)) cols)))))

(mu/defn matching-column-sets? :- :boolean
  "Returns true if the provided `refs` is the same set as the provided `columns`.
  Order is ignored. Only returns true if each of the `refs` matches a column, and each of the `columns` is matched by
  exactly 1 of the `refs`. (A bijection, in math terms.)"
  [query        :- ::lib.schema/query
   stage-number :- :int
   refs         :- [:sequential ::lib.schema.ref/ref]
   columns      :- [:sequential ::lib.schema.metadata/column]]
  ;; The lists match iff:
  ;; - Each ref matches a column; AND
  ;; - Each column was matched by exactly one ref
  ;; So we return true if nil is not a key in the matching, AND all vals in the matching have length 1,
  ;; AND the matching has as many elements as `columns` (usually the list of columns returned by default).
  (and (= (count refs) (count columns))
       (let [matching (group-by #(find-matching-column query stage-number % columns) refs)]
         (and (not (contains? matching nil))
              (= (count matching) (count columns))
              (every? #(= (count %) 1) (vals matching))))))

(ns metabase.util.format
  #?(:clj  (:require
            [colorize.core :as colorize]
            [metabase.config :as config])
     :cljs (:require
            [goog.string :as gstring])))

(defn- format-with-unit [n suffix]
  #?(:clj  (format "%.1f %s" n suffix)
     :cljs (str (.toFixed n 1) " " suffix)))

(defn format-nanoseconds
  ^String [nanoseconds]
  ;; The basic idea is to take `n` and see if it's greater than the divisior. If it is, we'll print it out as that
  ;; unit. If more, we'll divide by the divisor and recur, trying each successively larger unit in turn. e.g.
  ;;
  ;; (format-nanoseconds 500)    ; -> "500 ns"
  ;; (format-nanoseconds 500000) ; -> "500 µs"
  (loop [n nanoseconds, [[unit divisor] & more] [[:ns 1000] [:µs 1000] [:ms 1000] [:s 60] [:mins 60] [:hours 24]
                                                 [:days 7] [:weeks (/ 365.25 7)]
                                                 [:years #?(:clj  Double/POSITIVE_INFINITY
                                                            :cljs js/Number.POSITIVE_INFINITY)]]]
    (if (and (> n divisor)
             (seq more))
      (recur (/ n divisor) more)
      (format-with-unit (double n) (name unit)))))

(defn format-microseconds
  ^String [microseconds]
  (format-nanoseconds (* 1000.0 microseconds)))

(defn format-milliseconds
  ^String [milliseconds]
  (format-microseconds (* 1000.0 milliseconds)))

(defn format-seconds
  ^String [seconds]
  (format-milliseconds (* 1000.0 seconds)))

(defn format-bytes
  [num-bytes]
  (loop [n num-bytes [suffix & more] ["B" "KB" "MB" "GB"]]
    (if (and (seq more)
             (>= n 1024))
      (recur (/ n 1024.0) more)
      (format-with-unit n suffix))))

#?(:clj
   (def ^:private colorize?
     ;; As of 0.35.0 we support the NO_COLOR env var. See https://no-color.org/ (But who hates color logs?)
     (if (config/config-str :no-color)
       false
       (config/config-bool :mb-colorize-logs))))

(def ^{:arglists '(^String [color-symb x])} colorize
  #?(:clj  (if colorize?
             (fn [color x]
               (colorize/color (keyword color) (str x)))
             (fn [_ x]
               (str x)))
     :cljs (fn [_ x]
             (str x))))

(defn format-color
  {:arglists '(^String [color x] ^String [color format-string & args])}
  (^String [color x]
   (colorize color x))
  (^String [color format-str & args]
   (colorize color (apply #?(:clj format :cljs gstring/format) format-str args))))

(ns metabase.lib.schema.literal
  (:require
   #?@(:clj ([metabase.lib.schema.literal.jvm]))
   [malli.core :as mc]
   [metabase.lib.schema.common :as common]
   [metabase.lib.schema.expression :as expression]
   [metabase.lib.schema.mbql-clause :as mbql-clause]
   [metabase.shared.util.internal.time-common :as shared.ut.common]
   [metabase.util.malli.registry :as mr]))

(defmethod expression/type-of-method :dispatch-type/nil
  [_nil]
  :type/*)

(mr/def ::boolean
  :boolean)

(defmethod expression/type-of-method :dispatch-type/boolean
  [_bool]
  :type/Boolean)

(mr/def ::boolean
  :boolean)

(mr/def ::integer
  #?(:clj [:or
           :int
           :metabase.lib.schema.literal.jvm/big-integer]
     :cljs :int))

(defmethod expression/type-of-method :dispatch-type/integer
  [_int]
  :type/Integer)

(mr/def ::non-integer-real
  #?(:clj [:or
           :double
           :metabase.lib.schema.literal.jvm/float
           :metabase.lib.schema.literal.jvm/big-decimal]
     :cljs :double))

(defmethod expression/type-of-method :dispatch-type/number
  [_non-integer-real]
  ;; `:type/Float` is the 'base type' of all non-integer real number types in [[metabase.types]] =(
  :type/Float)

(mr/def ::string
  :string)

(mr/def ::string.date
  [:re
   {:error/message "date string literal"}
   shared.ut.common/local-date-regex])

(mr/def ::string.zone-offset
  [:re
   {:error/message "timezone offset string literal"}
   shared.ut.common/zone-offset-part-regex])

(mr/def ::string.time
  [:or
   [:re
    {:error/message "local time string literal"}
    shared.ut.common/local-time-regex]
   [:re
    {:error/message "offset time string literal"}
    shared.ut.common/offset-time-regex]])

(mr/def ::string.datetime
  [:or
   [:re
    {:error/message "local date time string literal"}
    shared.ut.common/local-datetime-regex]
   [:re
    {:error/message "offset date time string literal"}
    shared.ut.common/offset-datetime-regex]])

(defmethod expression/type-of-method :dispatch-type/string
  [s]
  (condp mc/validate s
    ::string.datetime #{:type/Text :type/DateTime}
    ::string.date     #{:type/Text :type/Date}
    ::string.time     #{:type/Text :type/Time}
    :type/Text))

(mr/def ::date
  #?(:clj  [:or
            [:time/local-date {:error/message "instance of java.time.LocalDate"}]
            ::string.date]
     :cljs ::string.date))

(mr/def ::time
  #?(:clj [:or
           ::string.time
           [:time/local-time {:error/message "instance of java.time.LocalTime"}]
           [:time/offset-time {:error/message "instance of java.time.OffsetTime"}]]
     :cljs ::string.time))

(mr/def ::datetime
  #?(:clj [:or
           ::string.datetime
           [:time/local-date-time {:error/message "instance of java.time.LocalDateTime"}]
           [:time/offset-date-time {:error/message "instance of java.time.OffsetDateTime"}]
           [:time/zoned-date-time {:error/message "instance of java.time.ZonedDateTime"}]]
     :cljs ::string.datetime))

(mr/def ::temporal
  [:or
   ::date
   ::time
   ::datetime])

(mr/def ::string.year-month
  [:re
   {:error/message "year-month string literal"}
   shared.ut.common/year-month-regex])

(mr/def ::string.year
  [:re
   {:error/message "year string literal"}
   shared.ut.common/year-regex])

(mr/def ::value.options
  [:merge
   [:ref ::common/options]
   [:map
    [:effective-type ::common/base-type]]])

(mbql-clause/define-mbql-clause :value
  [:tuple
   {:error/message "Value :value clause"}
   #_tag   [:= :value]
   #_opts  [:ref ::value.options]
   #_value any?])

(ns metabase.util.malli
  (:refer-clojure :exclude [fn defn defmethod])
  (:require
   #?@(:clj
       ([metabase.util.i18n]
        [metabase.util.malli.defn :as mu.defn]
        [metabase.util.malli.fn :as mu.fn]
        [net.cgrand.macrovich :as macros]
        [potemkin :as p]))
   [clojure.core :as core]
   [malli.core :as mc]
   [malli.destructure]
   [malli.error :as me]
   [malli.util :as mut]
   [metabase.shared.util.i18n :as i18n])
  #?(:cljs (:require-macros [metabase.util.malli])))

#?(:clj
   (p/import-vars
    [mu.fn fn]
    [mu.defn defn]))

(core/defn humanize-include-value
  [{:keys [value message]}]
  ;; TODO Should this be translated with more complete context? (tru "{0}, received: {1}" message (pr-str value))
  (str message ", " (i18n/tru "received") ": " (pr-str value)))

(core/defn explain
  [schema value]
  (-> (mc/explain schema value)
      (me/humanize {:wrap humanize-include-value})))

(def ^:private Schema
  [:and any?
   [:fn {:description "a malli schema"} mc/schema]])

(def localized-string-schema
  #?(:clj  [:fn {:error/message "must be a localized string"}
            metabase.util.i18n/localized-string?]
     ;; TODO Is there a way to check if a string is being localized in CLJS, by the `ttag`?
     ;; The compiler seems to just inline the translated strings with no annotation or wrapping.
     :cljs :string))

(metabase.util.malli/defn with-api-error-message
  {:style/indent [:form]}
  ([mschema :- Schema error-message :- localized-string-schema]
   (with-api-error-message mschema error-message error-message))
  ([mschema                :- :any
    description-message    :- localized-string-schema
    specific-error-message :- localized-string-schema]
   (mut/update-properties (mc/schema mschema) assoc
                          ;; override generic description in api docs and :errors key in API's response
                          :description description-message
                          ;; override generic description in :specific-errors key in API's response
                          :error/fn    (constantly specific-error-message))))

#?(:clj
   (defmacro disable-enforcement
     {:style/indent 0}
     [& body]
     (macros/case
       :clj
       `(binding [mu.fn/*enforce* false]
          ~@body)
       :cljs
       `(do ~@body))))

#?(:clj
   (defmacro -defmethod-clj
     [multifn dispatch-value & fn-tail]
     (let [dispatch-value-symb (gensym "dispatch-value-")
           error-context-symb  (gensym "error-context-")]
       `(let [~dispatch-value-symb ~dispatch-value
              ~error-context-symb  {:fn-name        '~(or (some-> (resolve multifn) symbol)
                                                          (symbol multifn))
                                    :dispatch-value ~dispatch-value-symb}
              f#                   ~(mu.fn/instrumented-fn-form error-context-symb (mu.fn/parse-fn-tail fn-tail))]
          (.addMethod ~(vary-meta multifn assoc :tag 'clojure.lang.MultiFn)
                      ~dispatch-value-symb
                      f#)))))
#?(:clj
   (defmacro -defmethod-cljs
     [multifn dispatch-value & fn-tail]
     `(core/defmethod ~multifn ~dispatch-value
        ~@(mu.fn/deparameterized-fn-tail (mu.fn/parse-fn-tail fn-tail)))))

#?(:clj
   (defmacro defmethod
     [multifn dispatch-value & fn-tail]
     (macros/case
       :clj  `(-defmethod-clj ~multifn ~dispatch-value ~@fn-tail)
       :cljs `(-defmethod-cljs ~multifn ~dispatch-value ~@fn-tail))))

#?(:clj
   (defn validate-throw
     [schema-or-validator value]
     (if-not ((if (fn? schema-or-validator)
                schema-or-validator
                (mc/validator schema-or-validator))
              value)
       (throw (ex-info "Value does not match schema" {:value value :schema schema-or-validator}))
       value)))

(ns metabase.actions
  (:require
   [clojure.spec.alpha :as s]
   [malli.core :as mc]
   [malli.error :as me]
   [metabase.api.common :as api]
   [metabase.driver :as driver]
   [metabase.lib.metadata :as lib.metadata]
   [metabase.mbql.normalize :as mbql.normalize]
   [metabase.mbql.schema :as mbql.s]
   [metabase.mbql.util :as mbql.u]
   [metabase.models :refer [Database]]
   [metabase.models.setting :as setting]
   [metabase.query-processor.error-type :as qp.error-type]
   [metabase.query-processor.middleware.permissions :as qp.perms]
   [metabase.query-processor.store :as qp.store]
   [metabase.util :as u]
   [metabase.util.i18n :as i18n]
   [toucan2.core :as t2]))

(setting/defsetting database-enable-actions
  (i18n/deferred-tru "Whether to enable Actions for a specific Database.")
  :default false
  :type :boolean
  :visibility :public
  :database-local :only)

(defmulti normalize-action-arg-map
  {:arglists '([action arg-map]), :added "0.44.0"}
  (fn [action _arg-map]
    (keyword action)))

(defmethod normalize-action-arg-map :default
  [_action arg-map]
  arg-map)

(defmulti action-arg-map-spec
  {:arglists '([action]), :added "0.44.0"}
  keyword)

(defmethod action-arg-map-spec :default
  [_action]
  any?)

(defmulti perform-action!*
  {:arglists '([driver action database arg-map]), :added "0.44.0"}
  (fn [driver action _database _arg-map]
    [(driver/dispatch-on-initialized-driver driver)
     (keyword action)])
  :hierarchy #'driver/hierarchy)

(defn- known-actions
  []
  (into #{}
        (comp (filter sequential?)
              (map second))
        (keys (methods perform-action!*))))

(defmethod perform-action!* :default
  [driver action _database _arg-map]
  (let [action        (keyword action)
        known-actions (known-actions)]
    ;; return 404 if the action doesn't exist.
    (when-not (contains? known-actions action)
      (throw (ex-info (i18n/tru "Unknown Action {0}. Valid Actions are: {1}"
                                action
                                (pr-str known-actions))
                      {:status-code 404})))
    ;; return 400 if the action does exist but is not supported by this DB
    (throw (ex-info (i18n/tru "Action {0} is not supported for {1} Databases."
                              action
                              (pr-str driver))
                    {:status-code 400}))))

(def ^:dynamic *misc-value-cache*
  nil)

(defn cached-value
  [unique-key value-thunk]
  (or (when *misc-value-cache*
        (get @*misc-value-cache* unique-key))
      (let [value (value-thunk)]
        (when *misc-value-cache*
          (swap! *misc-value-cache* assoc unique-key value))
        value)))

(defn check-actions-enabled-for-database!
  [{db-settings :settings db-id :id driver :engine db-name :name :as db}]
  (when-not (driver/database-supports? driver :actions db)
    (throw (ex-info (i18n/tru "{0} Database {1} does not support actions."
                              (u/qualified-name driver)
                              (format "%d %s" db-id (pr-str db-name)))
                    {:status-code 400, :database-id db-id})))
  (binding [setting/*database-local-values* db-settings]
    (when-not (database-enable-actions)
      (throw (ex-info (i18n/tru "Actions are not enabled.")
                      {:status-code 400, :database-id db-id}))))
  nil)

(defn- database-for-action [action-or-id]
  (t2/select-one Database {:select [:db.*]
                           :from   :action
                           :join   [[:report_card :card] [:= :card.id :action.model_id]
                                    [:metabase_database :db] [:= :db.id :card.database_id]]
                           :where  [:= :action.id (u/the-id action-or-id)]}))

(defn check-actions-enabled!
  [action-or-id]
  (check-actions-enabled-for-database! (api/check-404 (database-for-action action-or-id))))

(defn perform-action!
  [action arg-map]
  (let [action  (keyword action)
        spec    (action-arg-map-spec action)
        arg-map (normalize-action-arg-map action arg-map)]
    (when (s/invalid? (s/conform spec arg-map))
      (throw (ex-info (format "Invalid Action arg map for %s: %s" action (s/explain-str spec arg-map))
                      (s/explain-data spec arg-map))))
    (let [{driver :engine :as db} (api/check-404 (qp.store/with-metadata-provider (:database arg-map)
                                                   (lib.metadata/database (qp.store/metadata-provider))))]
      (check-actions-enabled-for-database! db)
      (binding [*misc-value-cache* (atom {})]
        (qp.perms/check-query-action-permissions* arg-map)
        (driver/with-driver driver
          (perform-action!* driver action db arg-map))))))

(s/def :actions.args/id
  (s/and integer? pos?))

(s/def :actions.args.common/database
  :actions.args/id)

(s/def :actions.args/common
  (s/keys :req-un [:actions.args.common/database]))

(s/def :actions.args.crud.row.common.query/source-table
  :actions.args/id)

(s/def :actions.args.crud.row.common/query
  (s/keys :req-un [:actions.args.crud.row.common.query/source-table]))

(s/def :actions.args.crud.row/common
  (s/merge
   :actions.args/common
   (s/keys :req-un [:actions.args.crud.row.common/query])))

(defn- normalize-as-mbql-query
  ([query]
   (let [query (mbql.normalize/normalize (assoc query :type :query))]
     (when-let [error (me/humanize (mc/explain mbql.s/Query query))]
       (throw (ex-info
               (i18n/tru "Invalid query: {0}" (pr-str error))
               {:status-code 400, :type qp.error-type/invalid-query})))
     query))
  ([query & {:keys [exclude]}]
   (let [query (update-keys query mbql.u/normalize-token)]
     (merge (select-keys query exclude)
            (normalize-as-mbql-query (apply dissoc query exclude))))))

(defmethod normalize-action-arg-map :row/create
  [_action query]
  (normalize-as-mbql-query query :exclude #{:create-row}))

(s/def :actions.args.crud.row.create/create-row
  (s/map-of keyword? any?))

(s/def :actions.args.crud/row.create
  (s/merge
   :actions.args.crud.row/common
   (s/keys :req-un [:actions.args.crud.row.create/create-row])))

(defmethod action-arg-map-spec :row/create
  [_action]
  :actions.args.crud/row.create)

(defmethod normalize-action-arg-map :row/update
  [_action query]
  (normalize-as-mbql-query query :exclude #{:update-row}))

(s/def :actions.args.crud.row.update.query/filter
  vector?) ; MBQL filter clause

(s/def :actions.args.crud.row.update/query
  (s/merge
   :actions.args.crud.row.common/query
   (s/keys :req-un [:actions.args.crud.row.update.query/filter])))

(s/def :actions.args.crud.row.update/update-row
  (s/map-of keyword? any?))

(s/def :actions.args.crud/row.update
  (s/merge
   :actions.args.crud.row/common
   (s/keys :req-un [:actions.args.crud.row.update/update-row
                    :actions.args.crud.row.update/query])))

(defmethod action-arg-map-spec :row/update
  [_action]
  :actions.args.crud/row.update)

(defmethod normalize-action-arg-map :row/delete
  [_action query]
  (normalize-as-mbql-query query))

(s/def :actions.args.crud.row.delete.query/filter
  vector?) ; MBQL filter clause

(s/def :actions.args.crud.row.delete/query
  (s/merge
   :actions.args.crud.row.common/query
   (s/keys :req-un [:actions.args.crud.row.delete.query/filter])))

(s/def :actions.args.crud/row.delete
  (s/merge
   :actions.args.crud.row/common
   (s/keys :req-un [:actions.args.crud.row.delete/query])))

(defmethod action-arg-map-spec :row/delete
  [_action]
  :actions.args.crud/row.delete)

(s/def :actions.args.crud.bulk.common/table-id
  :actions.args/id)

(s/def :actions.args.crud.bulk/rows
  (s/cat :rows (s/+ (s/map-of string? any?))))

(s/def :actions.args.crud.bulk/common
  (s/merge
   :actions.args/common
   (s/keys :req-un [:actions.args.crud.bulk.common/table-id
                    :actions.args.crud.bulk/rows])))

(defn- normalize-bulk-crud-action-arg-map
  [{:keys [database table-id], rows :arg, :as _arg-map}]
  {:type :query, :query {:source-table table-id}
   :database database, :table-id table-id, :rows (map #(update-keys % u/qualified-name) rows)})

(defmethod normalize-action-arg-map :bulk/create
  [_action arg-map]
  (normalize-bulk-crud-action-arg-map arg-map))

(defmethod action-arg-map-spec :bulk/create
  [_action]
  :actions.args.crud.bulk/common)

(defmethod normalize-action-arg-map :bulk/update
  [_action arg-map]
  (normalize-bulk-crud-action-arg-map arg-map))

(defmethod action-arg-map-spec :bulk/update
  [_action]
  :actions.args.crud.bulk/common)

(defmethod normalize-action-arg-map :bulk/delete
  [_action arg-map]
  (normalize-bulk-crud-action-arg-map arg-map))

(defmethod action-arg-map-spec :bulk/delete
  [_action]
  :actions.args.crud.bulk/common)

(ns metabase.actions.error)

(def violate-unique-constraint
  ::violate-unique-constraint)

(def violate-not-null-constraint
  ::violate-not-null-constraint)

(def violate-foreign-key-constraint
  ::violate-foreign-key-constraint)

(def incorrect-value-type
  ::incorrect-value-type)

(ns metabase.actions.execution
  (:require
   [clojure.set :as set]
   [medley.core :as m]
   [metabase.actions :as actions]
   [metabase.actions.http-action :as http-action]
   [metabase.analytics.snowplow :as snowplow]
   [metabase.api.common :as api]
   [metabase.models :refer [Card DashboardCard Database Table]]
   [metabase.models.action :as action]
   [metabase.models.persisted-info :as persisted-info]
   [metabase.models.query :as query]
   [metabase.query-processor :as qp]
   [metabase.query-processor.card :as qp.card]
   [metabase.query-processor.error-type :as qp.error-type]
   [metabase.query-processor.middleware.permissions :as qp.perms]
   [metabase.query-processor.writeback :as qp.writeback]
   [metabase.util :as u]
   [metabase.util.i18n :refer [tru]]
   [metabase.util.log :as log]
   [toucan2.core :as t2]))

(defn- execute-query-action!
  [{:keys [dataset_query model_id] :as action} request-parameters]
  (log/tracef "Executing action\n\n%s" (u/pprint-to-str action))
  (try
    (let [parameters (for [parameter (:parameters action)]
                       (assoc parameter :value (get request-parameters (:id parameter))))
          query (-> dataset_query
                    (update :type keyword)
                    (assoc :parameters parameters))]
      (log/debugf "Query (before preprocessing):\n\n%s" (u/pprint-to-str query))
      (binding [qp.perms/*card-id* model_id]
        (qp.writeback/execute-write-query! query)))
    (catch Throwable e
      (if (= (:type (u/all-ex-data e)) qp.error-type/missing-required-permissions)
        (api/throw-403 e)
        (throw (ex-info (format "Error executing Action: %s" (ex-message e))
                        {:action     action
                         :parameters request-parameters}
                        e))))))

(defn- implicit-action-table
  [card_id]
  (let [card (t2/select-one Card :id card_id)
        {:keys [table-id]} (query/query->database-and-table-ids (:dataset_query card))]
    (t2/hydrate (t2/select-one Table :id table-id) :fields)))

(defn- execute-custom-action [action request-parameters]
  (let [{action-type :type} action]
    (actions/check-actions-enabled! action)
    (let [model (t2/select-one Card :id (:model_id action))]
      (when (and (= action-type :query) (not= (:database_id model) (:database_id action)))
        ;; the above check checks the db of the model. We check the db of the query action here
        (actions/check-actions-enabled-for-database!
         (t2/select-one Database :id (:database_id action)))))
    (try
     (case action-type
       :query
       (execute-query-action! action request-parameters)
       :http
       (http-action/execute-http-action! action request-parameters))
     (catch Exception e
       (log/error e "Error executing action.")
       (if-let [ed (ex-data e)]
         (let [ed (cond-> ed
                    (and (nil? (:status-code ed))
                         (= (:type ed) :missing-required-permissions))
                    (assoc :status-code 403)
                    (nil? (:message ed))
                    (assoc :message (ex-message e)))]
           (if (= (ex-data e) ed)
             (throw e)
             (throw (ex-info (ex-message e) ed e))))
         {:body {:message (or (ex-message e) (tru "Error executing action."))}
          :status 500})))))

(defn- check-no-extra-parameters
  [request-parameters destination-param-ids]
  (let [extra-parameters (set/difference (set (keys request-parameters))
                                         (set destination-param-ids))]
    (api/check (empty? extra-parameters)
               400
               {:status-code            400
                :message                (tru "No destination parameter found for {0}. Found: {1}"
                                             (pr-str extra-parameters)
                                             (pr-str destination-param-ids))
                :type                   qp.error-type/invalid-parameter
                :parameters             request-parameters
                :destination-parameters destination-param-ids})))

(defn- build-implicit-query
  [{:keys [model_id parameters] :as _action} implicit-action request-parameters]
  (let [{database-id :db_id
         table-id :id :as table} (implicit-action-table model_id)
        table-fields             (:fields table)
        pk-fields                (filterv #(isa? (:semantic_type %) :type/PK) table-fields)
        slug->field-name         (->> table-fields
                                      (map (juxt (comp u/slugify :name) :name))
                                      (into {})
                                      (m/filter-keys (set (map :id parameters))))
        _                        (api/check (action/unique-field-slugs? table-fields)
                                   400
                                   (tru "Cannot execute implicit action on a table with ambiguous column names."))
        _                        (api/check (= (count pk-fields) 1)
                                   400
                                   (tru "Must execute implicit action on a table with a single primary key."))
        _                        (check-no-extra-parameters request-parameters (keys slug->field-name))
        pk-field                 (first pk-fields)
        ;; Ignore params with nil values; the client doesn't reliably omit blank, optional parameters from the
        ;; request. See discussion at #29049
        simple-parameters        (->> (update-keys request-parameters (comp keyword slug->field-name))
                                      (filter (fn [[_k v]] (some? v)))
                                      (into {}))
        pk-field-name            (keyword (:name pk-field))
        row-parameters           (cond-> simple-parameters
                                   (not= implicit-action :row/create) (dissoc pk-field-name))
        requires_pk              (contains? #{:row/delete :row/update} implicit-action)]
    (api/check (or (not requires_pk)
                   (some? (get simple-parameters pk-field-name)))
               400
               (tru "Missing primary key parameter: {0}"
                    (pr-str (u/slugify (:name pk-field)))))
    (cond->
      {:query {:database database-id,
               :type :query,
               :query {:source-table table-id}}
       :row-parameters row-parameters}
      requires_pk
      (assoc-in [:query :query :filter]
                [:= [:field (:id pk-field) nil] (get simple-parameters pk-field-name)])
      requires_pk
      (assoc :prefetch-parameters [{:target [:dimension [:field (:id pk-field) nil]]
                                    :type "id"
                                    :value [(get simple-parameters pk-field-name)]}]))))

(defn- execute-implicit-action
  [action request-parameters]
  (let [implicit-action (keyword (:kind action))
        {:keys [query row-parameters]} (build-implicit-query action implicit-action request-parameters)
        _ (api/check (or (= implicit-action :row/delete) (seq row-parameters))
                     400
                     (tru "Implicit parameters must be provided."))
        arg-map (cond-> query
                  (= implicit-action :row/create)
                  (assoc :create-row row-parameters)
                  (= implicit-action :row/update)
                  (assoc :update-row row-parameters))]
    (binding [qp.perms/*card-id* (:model_id action)]
      (actions/perform-action! implicit-action arg-map))))

(defn execute-action!
  [action request-parameters]
  (let [;; if a value is supplied for a hidden parameter, it should raise an error
        field-settings         (get-in action [:visualization_settings :fields])
        hidden-param-ids       (->> (vals field-settings)
                                    (filter :hidden)
                                    (map :id))
        destination-param-ids  (set/difference (set (map :id (:parameters action))) (set hidden-param-ids))
        _ (check-no-extra-parameters request-parameters destination-param-ids)
        ;; add default values for missing parameters (including hidden ones)
        all-param-ids          (set (map :id (:parameters action)))
        provided-param-ids     (set (keys request-parameters))
        missing-param-ids      (set/difference all-param-ids provided-param-ids)
        missing-param-defaults (into {}
                                     (keep (fn [param-id]
                                             (when-let [default-value (get-in field-settings [param-id :defaultValue])]
                                               [param-id default-value])))
                                     missing-param-ids)
        request-parameters     (merge missing-param-defaults request-parameters)]
    (case (:type action)
      :implicit
      (execute-implicit-action action request-parameters)
      (:query :http)
      (execute-custom-action action request-parameters)
      (throw (ex-info (tru "Unknown action type {0}." (name (:type action))) action)))))

(defn execute-dashcard!
  [dashboard-id dashcard-id request-parameters]
  (let [dashcard (api/check-404 (t2/select-one DashboardCard
                                               :id dashcard-id
                                               :dashboard_id dashboard-id))
        action (api/check-404 (action/select-action :id (:action_id dashcard)))]
    (snowplow/track-event! ::snowplow/action-executed api/*current-user-id* {:source    :dashboard
                                                                             :type      (:type action)
                                                                             :action_id (:id action)})
    (execute-action! action request-parameters)))

(defn- fetch-implicit-action-values
  [action request-parameters]
  (api/check (contains? #{"row/update" "row/delete"} (:kind action))
             400
             (tru "Values can only be fetched for actions that require a Primary Key."))
  (let [implicit-action (keyword (:kind action))
        {:keys [prefetch-parameters]} (build-implicit-query action implicit-action request-parameters)
        info {:executed-by api/*current-user-id*
              :context     :action
              :action-id   (:id action)}
        card (t2/select-one Card :id (:model_id action))
        ;; prefilling a form with day old data would be bad
        result (binding [persisted-info/*allow-persisted-substitution* false]
                 (qp/process-query-and-save-execution!
                  (qp.card/query-for-card card prefetch-parameters nil nil)
                  info))
        ;; only expose values for fields that are not hidden
        hidden-param-ids (keep #(when (:hidden %) (:id %))
                               (vals (get-in action [:visualization_settings :fields])))
        exposed-param-ids (-> (set (map :id (:parameters action)))
                              (set/difference (set hidden-param-ids)))]
    (m/filter-keys
      #(contains? exposed-param-ids %)
      (zipmap
        (map (comp u/slugify :name) (get-in result [:data :cols]))
        (first (get-in result [:data :rows]))))))

(defn fetch-values
  [action request-parameters]
  (if (= :implicit (:type action))
    (fetch-implicit-action-values action request-parameters)
    {}))

(ns metabase.actions.http-action
  (:require
   [cheshire.core :as json]
   [clj-http.client :as http]
   [clojure.string :as str]
   [metabase.driver.common.parameters :as params]
   [metabase.driver.common.parameters.parse :as params.parse]
   [metabase.query-processor.error-type :as qp.error-type]
   [metabase.util :as u]
   [metabase.util.i18n :refer [tru]]
   [metabase.util.log :as log])
  (:import
   (com.fasterxml.jackson.databind ObjectMapper)
   (net.thisptr.jackson.jq BuiltinFunctionLoader JsonQuery Output Scope Versions)))

(set! *warn-on-reflection* true)

(defonce ^:private root-scope
  (delay
    (let [scope (Scope/newEmptyScope)]
      (.loadFunctions (BuiltinFunctionLoader/getInstance) Versions/JQ_1_6 scope))))

(defonce ^:private object-mapper
  (delay (ObjectMapper.)))

(declare substitute*)

(defn- substitute-param [param->value [sql missing] _in-optional? {:keys [k]}]
  (if-not (contains? param->value k)
    [sql (conj missing k)]
    (let [v (get param->value k)]
      (cond
        (= params/no-value v)
        [sql (conj missing k)]
        :else
        [(str sql v) missing]))))

(defn- substitute-optional [param->value [sql missing] {subclauses :args}]
  (let [[opt-sql opt-missing] (substitute* param->value subclauses true)]
    (if (seq opt-missing)
      [sql missing]
      [(str sql opt-sql) missing])))

(defn- substitute*
  [param->value parsed in-optional?]
  (reduce
   (fn [[sql missing] x]
     (cond
       (string? x)
       [(str sql x) missing]
       (params/Param? x)
       (substitute-param param->value [sql missing] in-optional? x)
       (params/Optional? x)
       (substitute-optional param->value [sql missing] x)))
   nil
   parsed))

(defn substitute
  [parsed-template param->value]
  (log/tracef "Substituting params\n%s\nin template\n%s" (u/pprint-to-str param->value) (u/pprint-to-str parsed-template))
  (let [[sql missing] (try
                        (substitute* param->value parsed-template false)
                        (catch Throwable e
                          (throw (ex-info (tru "Unable to substitute parameters: {0}" (ex-message e))
                                          {:type         (or (:type (ex-data e)) qp.error-type/qp)
                                           :params       param->value
                                           :parsed-query parsed-template}
                                          e))))]
    (log/tracef "=>%s" sql)
    (when (seq missing)
      (throw (ex-info (tru "Cannot call the service: missing required parameters: {0}" (set missing))
                      {:type    qp.error-type/missing-required-parameter
                       :missing missing})))
    (str/trim sql)))

(defn- parse-and-substitute [s params->value]
  (when s
    (-> s
        params.parse/parse
        (substitute params->value))))

(deftype ActionOutput [results]
  Output
  (emit [_ x]
    (vswap! results conj (str x))))

(defn apply-json-query
  [object jq-query]
  ;; TODO this is pretty ineficient. We parse with `:as :json`, then reencode within a response
  ;; I couldn't find a way to get JSONNode out of cheshire, so we fall back to jackson.
  ;; Should jackson be added explicitly to deps.edn?
  (let [json-node (.readTree ^ObjectMapper @object-mapper (json/generate-string object))
        vresults (volatile! [])
        output (ActionOutput. vresults)
        expr (JsonQuery/compile jq-query Versions/JQ_1_6)
        ;; might need to Scope childScope = Scope.newChildScope(rootScope); if root-scope can be modified by expression
        _ (.apply expr @root-scope json-node output)
        results @vresults]
    (if (<= (count results) 1)
      (first results)
      (throw (ex-info (tru "Too many results returned: {0}" (pr-str results)) {:jq-query jq-query :results results})))))

(defn execute-http-action!
  [action params->value]
  (try
    (let [{:keys [method url body headers]} (:template action)
          request {:method (keyword method)
                   :url (parse-and-substitute url params->value)
                   :accept :json
                   :content-type :json
                   :throw-exceptions false
                   :headers (merge
                              ;; TODO maybe we want to default Agent here? Maybe Origin/Referer?
                              {"X-Metabase-Action" (:name action)}
                              (-> headers
                                  (parse-and-substitute params->value)
                                  (json/decode)))
                   :body (parse-and-substitute body params->value)}
          response (-> (http/request request)
                       (select-keys [:body :headers :status])
                       (update :body json/decode))
          error (json/parse-string (apply-json-query response (or (:error_handle action) ".status >= 400")))]
      (log/trace "Response before handle:" response)
      (if error
        {:status 400
         :headers {"Content-Type" "application/json"}
         :body (if (boolean? error)
                 {:remote-status (:status response)}
                 error)}
        (if-some [response (some->> action :response_handle (apply-json-query response))]
          {:status 200
           :headers {"Content-Type" "application/json"}
           :body response}
          {:status 204
           :body nil})))
    (catch Exception e
      (throw (ex-info (str "Problem building request: " (ex-message e))
                      {:template (:template action)}
                      e)))))

(ns metabase.analytics.prometheus
  (:refer-clojure :exclude [inc])
  (:require
   [clojure.java.jmx :as jmx]
   [iapetos.collector :as collector]
   [iapetos.collector.ring :as collector.ring]
   [iapetos.core :as prometheus]
   [metabase.models.setting :as setting :refer [defsetting]]
   [metabase.server :as server]
   [metabase.util.i18n :refer [deferred-trs trs]]
   [metabase.util.log :as log]
   [potemkin :as p]
   [potemkin.types :as p.types]
   [ring.adapter.jetty9 :as ring-jetty])
  (:import
   (io.prometheus.client Collector GaugeMetricFamily)
   (io.prometheus.client.hotspot GarbageCollectorExports MemoryPoolsExports StandardExports ThreadExports)
   (io.prometheus.client.jetty JettyStatisticsCollector)
   (java.util ArrayList List)
   (javax.management ObjectName)
   (org.eclipse.jetty.server Server)))

(set! *warn-on-reflection* true)

(defsetting prometheus-server-port
  (deferred-trs (str "Port to serve prometheus metrics from. If set, prometheus collectors are registered"
                     " and served from `localhost:<port>/metrics`."))
  :type       :integer
  :visibility :internal
  ;; settable only through environmental variable
  :setter     :none
  :getter     (fn reading-prometheus-port-setting []
                (let [parse (fn [raw-value]
                              (if-let [parsed (parse-long raw-value)]
                                parsed
                                (log/warn (trs "MB_PROMETHEUS_SERVER_PORT value of ''{0}'' is not parseable as an integer."
                                               raw-value))))]
                  (setting/get-raw-value :prometheus-server-port integer? parse))))

(p.types/defprotocol+ PrometheusActions
  (stop-web-server [this]))

(p/defrecord+ PrometheusSystem [registry web-server]
  ;; prometheus just runs in the background collecting metrics and serving them from
  ;; localhost:<prometheus-server-port>/metrics. Nothing we need to do but shutdown.
  PrometheusActions
  (stop-web-server [_this]
    (when-let [^Server web-server web-server]
      (.stop web-server))))

(defonce ^:private  ^PrometheusSystem system nil)

(declare setup-metrics! start-web-server!)

(defn- make-prometheus-system
  [port registry-name]
  (try
    (let [registry   (setup-metrics! registry-name)
          web-server (start-web-server! port registry)]
      (->PrometheusSystem registry web-server))
    (catch Exception e
      (throw (ex-info (trs "Failed to initialize Prometheus on port {0}" port)
                      {:port port}
                      e)))))

(defn c3p0-stats
  [raw-stats]
  (let [now    (.toEpochMilli (java.time.Instant/now))
        sample (fn sample [[db-label k v]]
                 {:name      k
                  :value     (double v)
                  :timestamp now
                  :label     db-label})]
    (->> raw-stats
         (mapcat (fn [[db-label values]]
                   (map (fn [[k v]] [db-label k v]) values)))
         (map sample)
         (group-by :name))))

(def ^:private label-translation
  {:maxPoolSize        {:label       "c3p0_max_pool_size"
                        :description (deferred-trs "C3P0 Max pool size")}
   :minPoolSize        {:label       "c3p0_min_pool_size"
                        :description (deferred-trs "C3P0 Minimum pool size")}
   :numConnections     {:label       "c3p0_num_connections"
                        :description (deferred-trs "C3P0 Number of connections")}
   :numIdleConnections {:label       "c3p0_num_idle_connections"
                        :description (deferred-trs "C3P0 Number of idle connections")}
   :numBusyConnections {:label       "c3p0_num_busy_connections"
                        :description (deferred-trs "C3P0 Number of busy connections")}
   :numThreadsAwaitingCheckoutDefaultUser
                       {:label       "c3p0_num_threads_awaiting_checkout_default_user"
                        :description (deferred-trs "C3P0 Number of threads awaiting checkout")}})

(defn- stats->prometheus
  [stats]
  (let [arr (ArrayList. (count stats))]
    (doseq [[raw-label measurements] stats]
      (if-let [{gauge-label :label desc :description} (label-translation raw-label)]
        (let [gauge (GaugeMetricFamily.
                     ^String gauge-label
                     ^String (str desc) ;; site-localized becomes string
                     (List/of "database"))]
          (doseq [m measurements]
            (.addMetric gauge (List/of (:label m)) (:value m)))
          (.add arr gauge))
        (log/warn (trs "Unrecognized measurement {0} in prometheus stats"
                       raw-label))))
    arr))

(defn- conn-pool-bean-diag-info [acc ^ObjectName jmx-bean]
  (let [bean-id   (.getCanonicalName jmx-bean)
        props     [:numConnections :numIdleConnections :numBusyConnections
                   :minPoolSize :maxPoolSize :numThreadsAwaitingCheckoutDefaultUser]]
    (assoc acc (jmx/read bean-id :dataSourceName) (jmx/read bean-id props))))

(defn connection-pool-info
  []
  (reduce conn-pool-bean-diag-info {} (jmx/mbean-names "com.mchange.v2.c3p0:type=PooledDataSource,*")))

(def c3p0-collector
  (letfn [(collect-metrics []
            (-> (connection-pool-info)
                c3p0-stats
                stats->prometheus))]
    (delay
      (collector/named
       {:name "c3p0-stats"
        :namespace "metabase_database"}
       (proxy [Collector] []
         (collect
           ([] (collect-metrics))
           ([_sampleNameFilter] (collect-metrics))))))))

(defn- jvm-collectors
  []
  [(collector/named {:namespace "metabase_application"
                     :name      "jvm_gc"}
                    (GarbageCollectorExports.))
   (collector/named {:namespace "metabase_application"
                     :name      "jvm_standard"}
                    (StandardExports.))
   (collector/named {:namespace "metabase_application"
                     :name      "jvm_memory_pools"}
                    (MemoryPoolsExports.))
   (collector/named {:namespace "metabase_application"
                     :name      "jvm_threads"}
                    (ThreadExports.))])

(defn- jetty-collectors
  []
  ;; when in dev you might not have a server setup
  (when (server/instance)
    [(collector/named {:namespace "metabase_webserver"
                       :name      "jetty_stats"}
                      (JettyStatisticsCollector. (.getHandler (server/instance))))]))

(defn- setup-metrics!
  [registry-name]
  (log/info (trs "Starting prometheus metrics collector"))
  (let [registry (prometheus/collector-registry registry-name)]
    (apply prometheus/register registry
           (concat (jvm-collectors)
                   (jetty-collectors)
                   [@c3p0-collector]
                   ; Iapetos will use "default" if we do not provide a namespace, so explicitly set `metabase-email`:
                   [(prometheus/counter :metabase-email/messages
                                        {:description (trs "Number of emails sent.")})
                    (prometheus/counter :metabase-email/message-errors
                                        {:description (trs "Number of errors when sending emails.")})]))))

(defn- start-web-server!
  [port registry]
  (log/info (trs "Starting prometheus metrics web-server on port {0}" (str port)))
  (when-not port
    (throw (ex-info (trs "Attempting to set up prometheus metrics web-server with no web-server port provided")
                    {})))
  (ring-jetty/run-jetty (-> (constantly {:status 200})
                            (collector.ring/wrap-metrics registry {:path "/metrics"}))
                        {:join?       false
                         :port        port
                         :max-threads 8}))

(defn setup!
  []
  (let [port (prometheus-server-port)]
    (when-not port
      (throw (ex-info (trs "Attempting to set up prometheus metrics with no web-server port provided")
                      {})))
    (when-not system
      (locking #'system
        (when-not system
          (let [sys (make-prometheus-system port "metabase-registry")]
            (alter-var-root #'system (constantly sys))))))))

(defn shutdown!
  []
  (when system
    (locking #'system
      (when system
        (try (stop-web-server system)
             (prometheus/clear (.-registry system))
             (alter-var-root #'system (constantly nil))
             (log/info (trs "Prometheus web-server shut down"))
             (catch Exception e
               (log/warn e (trs "Error stopping prometheus web-server"))))))))

(defn inc
  [metric]
  (some-> system .-registry metric prometheus/inc))

(comment
  (require 'iapetos.export)
  (spit "metrics" (iapetos.export/text-format (.registry system))))

(ns metabase.analytics.snowplow
  (:require
   [clojure.string :as str]
   [java-time.api :as t]
   [medley.core :as m]
   [metabase.config :as config]
   [metabase.models.setting :as setting :refer [defsetting Setting]]
   [metabase.models.user :refer [User]]
   [metabase.public-settings :as public-settings]
   [metabase.util.date-2 :as u.date]
   [metabase.util.i18n :refer [deferred-tru trs]]
   [metabase.util.log :as log]
   [toucan2.core :as t2])
  (:import
   (com.snowplowanalytics.snowplow.tracker Snowplow Subject Tracker)
   (com.snowplowanalytics.snowplow.tracker.configuration EmitterConfiguration NetworkConfiguration SubjectConfiguration TrackerConfiguration)
   (com.snowplowanalytics.snowplow.tracker.events SelfDescribing SelfDescribing$Builder2)
   (com.snowplowanalytics.snowplow.tracker.http ApacheHttpClientAdapter)
   (com.snowplowanalytics.snowplow.tracker.payload SelfDescribingJson)
   (org.apache.http.client.config CookieSpecs RequestConfig)
   (org.apache.http.impl.client HttpClients)
   (org.apache.http.impl.conn PoolingHttpClientConnectionManager)))

(set! *warn-on-reflection* true)

(def ^:private schema->version
  {::account      "1-0-1"
   ::invite       "1-0-1"
   ::csvupload    "1-0-0"
   ::dashboard    "1-1-3"
   ::database     "1-0-1"
   ::instance     "1-1-2"
   ::metabot      "1-0-1"
   ::search       "1-0-1"
   ::model        "1-0-0"
   ::timeline     "1-0-0"
   ::task         "1-0-0"
   ::action       "1-0-0"
   ::embed_share  "1-0-0"})

(def ^:private event->schema
  {::new-instance-created           ::account
   ::new-user-created               ::account
   ::invite-sent                    ::invite
   ::index-model-entities-enabled   ::model
   ::dashboard-created              ::dashboard
   ::question-added-to-dashboard    ::dashboard
   ::dashboard-tab-created          ::dashboard
   ::dashboard-tab-deleted          ::dashboard
   ::database-connection-successful ::database
   ::database-connection-failed     ::database
   ::new-event-created              ::timeline
   ::new-task-history               ::task
   ::new-search-query               ::search
   ::search-results-filtered        ::search
   ::action-created                 ::action
   ::action-updated                 ::action
   ::action-deleted                 ::action
   ::action-executed                ::action
   ::csv-upload-successful          ::csvupload
   ::csv-upload-failed              ::csvupload
   ::metabot-feedback-received      ::metabot
   ::embedding-enabled              ::embed_share
   ::embedding-disabled             ::embed_share})

(defsetting analytics-uuid
  (deferred-tru
    (str "Unique identifier to be used in Snowplow analytics, to identify this instance of Metabase. "
         "This is a public setting since some analytics events are sent prior to initial setup."))
  :visibility :public
  :setter     :none
  :type       ::public-settings/uuid-nonce
  :doc        false)

(defsetting snowplow-available
  (deferred-tru
   (str "Boolean indicating whether a Snowplow collector is available to receive analytics events. "
        "Should be set via environment variable in Cypress tests or during local development."))
  :type       :boolean
  :visibility :public
  :default    config/is-prod?
  :doc        false
  :audit      :never)

(defsetting snowplow-enabled
  (deferred-tru
   (str "Boolean indicating whether analytics events are being sent to Snowplow. "
        "True if anonymous tracking is enabled for this instance, and a Snowplow collector is available."))
  :type       :boolean
  :setter     :none
  :getter     (fn [] (and (snowplow-available)
                          (public-settings/anon-tracking-enabled)))
  :visibility :public
  :doc        false)

(defsetting snowplow-url
  (deferred-tru "The URL of the Snowplow collector to send analytics events to.")
  :default    (if config/is-prod?
                "https://sp.metabase.com"
                ;; See the iglu-schema-registry repo for instructions on how to run Snowplow Micro locally for development
                "http://localhost:9090")
  :visibility :public
  :audit      :never
  :doc        false)

(defn- first-user-creation
  []
  (:min (t2/select-one [User [:%min.date_joined :min]])))

(declare track-event!)

(defsetting instance-creation
  (deferred-tru "The approximate timestamp at which this instance of Metabase was created, for inclusion in analytics.")
  :visibility :public
  :setter     :none
  :getter     (fn []
                (when-not (t2/exists? Setting :key "instance-creation")
                  ;; For instances that were started before this setting was added (in 0.41.3), use the creation
                  ;; timestamp of the first user. For all new instances, use the timestamp at which this setting
                  ;; is first read.
                  (let [value (or (first-user-creation) (t/offset-date-time))]
                    (setting/set-value-of-type! :timestamp :instance-creation value)
                    (track-event! ::new-instance-created)))
                (u.date/format-rfc3339 (setting/get-value-of-type :timestamp :instance-creation)))
  :doc false)

(def ^:private tracker-config
  (let [tracker-config* (delay (TrackerConfiguration. "sp" "metabase"))]
    (fn [] @tracker-config*)))

(def ^:private network-config
  (let [network-config* (delay
                         (let [request-config (-> (RequestConfig/custom)
                                                  ;; Set cookie spec to `STANDARD` to avoid warnings about an invalid cookie
                                                  ;; header in request response (PR #24579)
                                                  (.setCookieSpec CookieSpecs/STANDARD)
                                                  (.build))
                               client (-> (HttpClients/custom)
                                          (.setConnectionManager (PoolingHttpClientConnectionManager.))
                                          (.setDefaultRequestConfig request-config)
                                          (.build))
                               http-client-adapter (ApacheHttpClientAdapter. (snowplow-url) client)]
                           (NetworkConfiguration. http-client-adapter)))]
    (fn [] @network-config*)))

(def ^:private emitter-config
  (let [emitter-config* (delay (-> (EmitterConfiguration.)
                                   (.batchSize 1)))]
     (fn [] @emitter-config*)))

(def ^:private tracker
  (let [tracker* (delay
                  (Snowplow/createTracker
                   ^TrackerConfiguration (tracker-config)
                   ^NetworkConfiguration (network-config)
                   ^EmitterConfiguration (emitter-config)))]
    (fn [] @tracker*)))

(defn- subject
  [user-id]
  (Subject.
   (-> (SubjectConfiguration.)
       (.userId (str user-id))
       ;; Override with localhost IP to avoid logging actual user IP addresses
       (.ipAddress "127.0.0.1"))))

(defn- app-db-type
  []
  (t2/with-connection [^java.sql.Connection conn]
    (.. conn getMetaData getDatabaseProductName)))

(defn- app-db-version
  []
  (t2/with-connection [^java.sql.Connection conn]
    (let [metadata (.getMetaData conn)]
      (format "%d.%d" (.getDatabaseMajorVersion metadata) (.getDatabaseMinorVersion metadata)))))

(defn- context
  []
  (new SelfDescribingJson
       (str "iglu:com.metabase/instance/jsonschema/" (schema->version ::instance))
       {"id"                           (analytics-uuid)
        "version"                      {"tag" (:tag (public-settings/version))}
        "token_features"               (m/map-keys name (public-settings/token-features))
        "created_at"                   (instance-creation)
        "application_database"         (app-db-type)
        "application_database_version" (app-db-version)}))

(defn- normalize-kw
  [kw]
  (-> kw name (str/replace #"-" "_")))

(defn- payload
  [schema version event-kw data]
  (new SelfDescribingJson
       (format "iglu:com.metabase/%s/jsonschema/%s" (normalize-kw schema) version)
       ;; Make sure keywords in payload are converted to strings in snake-case
       (m/map-kv
        (fn [k v] [(normalize-kw k) (if (keyword? v) (normalize-kw v) v)])
        (assoc data :event event-kw))))

(defn- track-event-impl!
  [tracker event]
  (.track ^Tracker tracker ^SelfDescribing event))

(defn track-event!
  [event-kw & [user-id data]]
  (when (snowplow-enabled)
    (try
      (let [schema (event->schema event-kw)
            ^SelfDescribing$Builder2 builder (-> (. SelfDescribing builder)
                                                 (.eventData (payload schema (schema->version schema) event-kw data))
                                                 (.customContext [(context)])
                                                 (cond-> user-id (.subject (subject user-id))))
            ^SelfDescribing event (.build builder)]
        (track-event-impl! (tracker) event))
      (catch Throwable e
        (log/error e (trs "Error sending Snowplow analytics event {0}" event-kw))))))

(ns metabase.analytics.stats
  (:require
   [cheshire.core :as json]
   [clj-http.client :as http]
   [clojure.string :as str]
   [java-time.api :as t]
   [medley.core :as m]
   [metabase.analytics.snowplow :as snowplow]
   [metabase.config :as config]
   [metabase.db.query :as mdb.query]
   [metabase.db.util :as mdb.u]
   [metabase.driver :as driver]
   [metabase.email :as email]
   [metabase.embed.settings :as embed.settings]
   [metabase.integrations.google :as google]
   [metabase.integrations.slack :as slack]
   [metabase.models
    :refer [Card
            Collection
            Dashboard
            DashboardCard
            Database
            Field
            Metric
            PermissionsGroup
            Pulse
            PulseCard
            PulseChannel
            QueryCache
            Segment
            Table
            User]]
   [metabase.models.humanization :as humanization]
   [metabase.public-settings :as public-settings]
   [metabase.util :as u]
   [metabase.util.honey-sql-2 :as h2x]
   [metabase.util.i18n :refer [trs]]
   [metabase.util.log :as log]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(defn- merge-count-maps
  [ms]
  (reduce (partial merge-with +)
          {}
          (for [m ms]
            (m/map-vals #(cond
                           (number? %) %
                           %           1
                           :else       0)
                        m))))

(def ^:private ^String metabase-usage-url "https://xuq0fbkk0j.execute-api.us-east-1.amazonaws.com/prod")

(defn- bin-small-number
  [x]
  (cond
    (= 0 x)      "0"
    (<= 1 x 5)   "1-5"
    (<= 6 x 10)  "6-10"
    (<= 11 x 25) "11-25"
    (> x 25)     "25+"))

(defn- bin-medium-number
  [x]
  (cond
    (= 0 x)        "0"
    (<= 1 x 5)     "1-5"
    (<= 6 x 10)    "6-10"
    (<= 11 x 25)   "11-25"
    (<= 26 x 50)   "26-50"
    (<= 51 x 100)  "51-100"
    (<= 101 x 250) "101-250"
    (> x 250)      "250+"))

(defn- bin-large-number
  [x]
  (cond
    (= 0 x)           "0"
    (< x 1)           "< 1"
    (<= 1 x 10)       "1-10"
    (<= 11 x 50)      "11-50"
    (<= 51 x 250)     "51-250"
    (<= 251 x 1000)   "251-1000"
    (<= 1001 x 10000) "1001-10000"
    (> x 10000)       "10000+"))

(defn- value-frequencies
  [many-maps k]
  (frequencies (map k many-maps)))

(defn- histogram
  ([binning-fn counts]
   (frequencies (map binning-fn counts)))
  ([binning-fn many-maps k]
   (histogram binning-fn (vals (value-frequencies many-maps k)))))

(def ^:private medium-histogram
  (partial histogram bin-medium-number))

(defn environment-type
  []
  (cond
    (config/config-str :rds-hostname)        :elastic-beanstalk
    (config/config-str :database-url)        :heroku ;; Putting this last as 'database-url' seems least specific
    :else                                    :unknown))

(defn- instance-settings
  []
  {:version                  (config/mb-version-info :tag)
   :running_on               (environment-type)
   :startup_time_millis      (public-settings/startup-time-millis)
   :application_database     (config/config-str :mb-db-type)
   :check_for_updates        (public-settings/check-for-updates)
   :site_name                (not= (public-settings/site-name) "Metabase")
   :report_timezone          (driver/report-timezone)
   ; We deprecated advanced humanization but have this here anyways
   :friendly_names           (= (humanization/humanization-strategy) "advanced")
   :email_configured         (email/email-configured?)
   :slack_configured         (slack/slack-configured?)
   :sso_configured           (google/google-auth-enabled)
   :instance_started         (snowplow/instance-creation)
   :has_sample_data          (t2/exists? Database, :is_sample true)
   :help_link                (public-settings/help-link)
   :enable_embedding         (embed.settings/enable-embedding)
   :embedding_app_origin_set (boolean (embed.settings/embedding-app-origin))})

(defn- user-metrics
  []
  {:users (merge-count-maps (for [user (t2/select [User :is_active :is_superuser :last_login :sso_source]
                                                  :type :personal)]
                              {:total     1
                               :active    (:is_active    user)
                               :admin     (:is_superuser user)
                               :logged_in (:last_login   user)
                               :sso       (= :google (:sso_source user))}))})

(defn- group-metrics
  []
  {:groups (t2/count PermissionsGroup)})

(defn- card-has-params? [card]
  (boolean (get-in card [:dataset_query :native :template-tags])))

(defn- question-metrics
  []
  (let [cards (t2/select [Card :query_type :public_uuid :enable_embedding :embedding_params :dataset_query])]
    {:questions (merge-count-maps (for [card cards]
                                    (let [native? (= (keyword (:query_type card)) :native)]
                                      {:total       1
                                       :native      native?
                                       :gui         (not native?)
                                       :with_params (card-has-params? card)})))
     :public    (merge-count-maps (for [card  cards
                                        :when (:public_uuid card)]
                                    {:total       1
                                     :with_params (card-has-params? card)}))
     :embedded  (merge-count-maps (for [card  cards
                                        :when (:enable_embedding card)]
                                    (let [embedding-params-vals (set (vals (:embedding_params card)))]
                                      {:total                1
                                       :with_params          (card-has-params? card)
                                       :with_enabled_params  (contains? embedding-params-vals "enabled")
                                       :with_locked_params   (contains? embedding-params-vals "locked")
                                       :with_disabled_params (contains? embedding-params-vals "disabled")})))}))

(defn- dashboard-metrics
  []
  (let [dashboards (t2/select [Dashboard :creator_id :public_uuid :parameters :enable_embedding :embedding_params])
        dashcards  (t2/select [DashboardCard :card_id :dashboard_id])]
    {:dashboards         (count dashboards)
     :with_params        (count (filter (comp seq :parameters) dashboards))
     :num_dashs_per_user (medium-histogram dashboards :creator_id)
     :num_cards_per_dash (medium-histogram dashcards :dashboard_id)
     :num_dashs_per_card (medium-histogram dashcards :card_id)
     :public             (merge-count-maps (for [dash  dashboards
                                                 :when (:public_uuid dash)]
                                             {:total       1
                                              :with_params (seq (:parameters dash))}))
     :embedded           (merge-count-maps (for [dash  dashboards
                                                 :when (:enable_embedding dash)]
                                             (let [embedding-params-vals (set (vals (:embedding_params dash)))]
                                               {:total                1
                                                :with_params          (seq (:parameters dash))
                                                :with_enabled_params  (contains? embedding-params-vals "enabled")
                                                :with_locked_params   (contains? embedding-params-vals "locked")
                                                :with_disabled_params (contains? embedding-params-vals "disabled")})))}))

(defn- db-frequencies
  {:style/indent 2}
  [model column & [additonal-honeysql]]
  (into {} (for [{:keys [k count]} (t2/select [model [column :k] [:%count.* :count]]
                                     (merge {:group-by [column]}
                                            additonal-honeysql))]
             [k count])))

(defn- num-notifications-with-xls-or-csv-cards
  [& where-conditions]
  (-> (mdb.query/query {:select    [[[::h2x/distinct-count :pulse.id] :count]]
                        :from      [:pulse]
                        :left-join [:pulse_card [:= :pulse.id :pulse_card.pulse_id]]
                        :where     (into
                                    [:and
                                     [:or
                                      [:= :pulse_card.include_csv true]
                                      [:= :pulse_card.include_xls true]]]
                                    where-conditions)})
      first
      :count))

(defn- pulse-metrics
  []
  (let [pulse-conditions {:left-join [:pulse [:= :pulse.id :pulse_id]], :where [:= :pulse.alert_condition nil]}]
    {:pulses               (t2/count Pulse :alert_condition nil)
     ;; "Table Cards" are Cards that include a Table you can download
     :with_table_cards     (num-notifications-with-xls-or-csv-cards [:= :alert_condition nil])
     :pulse_types          (db-frequencies PulseChannel :channel_type  pulse-conditions)
     :pulse_schedules      (db-frequencies PulseChannel :schedule_type pulse-conditions)
     :num_pulses_per_user  (medium-histogram (vals (db-frequencies Pulse     :creator_id (dissoc pulse-conditions :left-join))))
     :num_pulses_per_card  (medium-histogram (vals (db-frequencies PulseCard :card_id    pulse-conditions)))
     :num_cards_per_pulses (medium-histogram (vals (db-frequencies PulseCard :pulse_id   pulse-conditions)))}))

(defn- alert-metrics []
  (let [alert-conditions {:left-join [:pulse [:= :pulse.id :pulse_id]], :where [:not= (mdb.u/qualify Pulse :alert_condition) nil]}]
    {:alerts               (t2/count Pulse :alert_condition [:not= nil])
     :with_table_cards     (num-notifications-with-xls-or-csv-cards [:not= :alert_condition nil])
     :first_time_only      (t2/count Pulse :alert_condition [:not= nil], :alert_first_only true)
     :above_goal           (t2/count Pulse :alert_condition [:not= nil], :alert_above_goal true)
     :alert_types          (db-frequencies PulseChannel :channel_type alert-conditions)
     :num_alerts_per_user  (medium-histogram (vals (db-frequencies Pulse     :creator_id (dissoc alert-conditions :left-join))))
     :num_alerts_per_card  (medium-histogram (vals (db-frequencies PulseCard :card_id    alert-conditions)))
     :num_cards_per_alerts (medium-histogram (vals (db-frequencies PulseCard :pulse_id   alert-conditions)))}))

(defn- collection-metrics
  []
  (let [collections (t2/select Collection)
        cards       (t2/select [Card :collection_id])]
    {:collections              (count collections)
     :cards_in_collections     (count (filter :collection_id cards))
     :cards_not_in_collections (count (remove :collection_id cards))
     :num_cards_per_collection (medium-histogram cards :collection_id)}))

(defn- database-metrics
  []
  (let [databases (t2/select [Database :is_full_sync :engine :dbms_version])]
    {:databases (merge-count-maps (for [{is-full-sync? :is_full_sync} databases]
                                    {:total    1
                                     :analyzed is-full-sync?}))
     :dbms_versions (frequencies (map (fn [db]
                                        (-> db
                                            :dbms_version
                                            (assoc :engine (:engine db))
                                            json/generate-string))
                                      databases))}))

(defn- table-metrics
  []
  (let [tables (t2/select [Table :db_id :schema])]
    {:tables           (count tables)
     :num_per_database (medium-histogram tables :db_id)
     :num_per_schema   (medium-histogram tables :schema)}))

(defn- field-metrics
  []
  (let [fields (t2/select [Field :table_id])]
    {:fields        (count fields)
     :num_per_table (medium-histogram fields :table_id)}))

(defn- segment-metrics
  []
  {:segments (t2/count Segment)})

(defn- metric-metrics
  []
  {:metrics (t2/count Metric)})

(defn summarize-executions
  ([]
   (summarize-executions (t2/reducible-select [:model/QueryExecution :executor_id :running_time :error])))
  ([executions]
   (reduce summarize-executions {:executions 0, :by_status {}, :num_per_user {}, :num_by_latency {}} executions))
  ([summary execution]
   (-> summary
       (update :executions u/safe-inc)
       (update-in [:by_status (if (:error execution)
                                "failed"
                                "completed")] u/safe-inc)
       (update-in [:num_per_user (:executor_id execution)] u/safe-inc)
       (update-in [:num_by_latency (bin-large-number (/ (:running_time execution) 1000))] u/safe-inc))))

(defn- summarize-executions-per-user
  [user-id->num-executions]
  (frequencies (map bin-large-number (vals user-id->num-executions))))

(defn- execution-metrics
  []
  (-> (summarize-executions)
      (update :num_per_user summarize-executions-per-user)))

(defn- cache-metrics
  []
  (let [{:keys [length count]} (t2/select-one [QueryCache [[:avg [:length :results]] :length] [:%count.* :count]])]
    {:average_entry_size (int (or length 0))
     :num_queries_cached (bin-small-number count)}))

(defn- bytes->megabytes [b]
  (Math/round (double (/ b 1024 1024))))

(def ^:private system-property-names
  ["java.version" "java.vm.specification.version"  "java.runtime.name"
   "user.timezone" "user.language" "user.country" "file.encoding"
   "os.name" "os.version"])

(defn- system-metrics
  []
  (let [runtime (Runtime/getRuntime)]
    (merge
     {:max_memory (bytes->megabytes (.maxMemory runtime))
      :processors (.availableProcessors runtime)}
     (zipmap (map #(keyword (str/replace % \. \_)) system-property-names)
             (map #(System/getProperty %) system-property-names)))))

(defn anonymous-usage-stats
  []
  (merge (instance-settings)
         {:uuid      (public-settings/site-uuid)
          :timestamp (t/offset-date-time)
          :stats     {:cache      (cache-metrics)
                      :collection (collection-metrics)
                      :dashboard  (dashboard-metrics)
                      :database   (database-metrics)
                      :execution  (execution-metrics)
                      :field      (field-metrics)
                      :group      (group-metrics)
                      :metric     (metric-metrics)
                      :pulse      (pulse-metrics)
                      :alert      (alert-metrics)
                      :question   (question-metrics)
                      :segment    (segment-metrics)
                      :system     (system-metrics)
                      :table      (table-metrics)
                      :user       (user-metrics)}}))

(defn- send-stats!
  [stats]
  (try
     (http/post metabase-usage-url {:form-params stats, :content-type :json, :throw-entire-message? true})
     (catch Throwable e
       (log/error e (trs "Sending usage stats FAILED")))))

(defn phone-home-stats!
  []
  (when (public-settings/anon-tracking-enabled)
    (send-stats! (anonymous-usage-stats))))

(ns metabase.api.action
  (:require
   [cheshire.core :as json]
   [compojure.core :as compojure :refer [POST]]
   [metabase.actions :as actions]
   [metabase.actions.execution :as actions.execution]
   [metabase.actions.http-action :as http-action]
   [metabase.analytics.snowplow :as snowplow]
   [metabase.api.common :as api]
   [metabase.api.common.validation :as validation]
   [metabase.models :refer [Action Card Database]]
   [metabase.models.action :as action]
   [metabase.models.card :as card]
   [metabase.models.collection :as collection]
   [metabase.util :as u]
   [metabase.util.i18n :refer [deferred-tru tru]]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(def ^:private json-query-schema
  [:and
   string?
   (mu/with-api-error-message
     [:fn #(http-action/apply-json-query {} %)]
     (deferred-tru "must be a valid json-query, something like ''.item.title''"))])

(def ^:private supported-action-type
  (mu/with-api-error-message
    [:enum "http" "query" "implicit"]
    (deferred-tru "Unsupported action type")))

(def ^:private implicit-action-kind
  (mu/with-api-error-message
    (into [:enum]
          (for [ns ["row" "bulk"]
                action ["create" "update" "delete"]]
            (str ns "/" action)))
    (deferred-tru "Unsupported implicit action kind")))

(def ^:private http-action-template
  [:map {:closed true}
   [:method                              [:enum "GET" "POST" "PUT" "DELETE" "PATCH"]]
   [:url                                 [string? {:min 1}]]
   [:body               {:optional true} [:maybe string?]]
   [:headers            {:optional true} [:maybe string?]]
   [:parameters         {:optional true} [:maybe [:sequential map?]]]
   [:parameter_mappings {:optional true} [:maybe map?]]])

(api/defendpoint GET 
  "Returns actions that can be used for QueryActions. By default lists all viewable actions. Pass optional
  `?model-id=<model-id>` to limit to actions on a particular model."
  [model-id]
  {model-id [:maybe ms/PositiveInt]}
  (letfn [(actions-for [models]
            (if (seq models)
              (t2/hydrate (action/select-actions models
                                              :model_id [:in (map :id models)]
                                              :archived false)
                       :creator)
              []))]
    ;; We don't check the permissions on the actions, we assume they are readable if the model is readable.
    (let [models (if model-id
                   [(api/read-check Card model-id)]
                   (t2/select Card {:where
                                    [:and
                                     [:= :dataset true]
                                     [:= :archived false]
                                     ;; action permission keyed off of model permission
                                     (collection/visible-collection-ids->honeysql-filter-clause
                                      :collection_id
                                      (collection/permissions-set->visible-collection-ids
                                       @api/*current-user-permissions-set*))]}))]
      (actions-for models))))

(api/defendpoint GET 
  "Fetch a list of Actions with public UUIDs. These actions are publicly-accessible *if* public sharing is enabled."
  []
  (validation/check-has-application-permission :setting)
  (validation/check-public-sharing-enabled)
  (t2/select [Action :name :id :public_uuid :model_id], :public_uuid [:not= nil], :archived false))

(api/defendpoint GET 
  [action-id]
  {action-id ms/PositiveInt}
  (-> (action/select-action :id action-id :archived false)
      (t2/hydrate :creator)
      api/read-check))

(api/defendpoint DELETE 
  [action-id]
  {action-id ms/PositiveInt}
  (let [action (api/write-check Action action-id)]
    (snowplow/track-event! ::snowplow/action-deleted api/*current-user-id* {:type      (:type action)
                                                                            :action_id action-id}))
  (t2/delete! Action :id action-id)
  api/generic-204-no-content)

(api/defendpoint POST 
  "Create a new action."
  [:as {{:keys [type name description model_id parameters parameter_mappings visualization_settings
                kind
                database_id dataset_query
                template response_handle error_handle] :as action} :body}]
  {name                   :string
   model_id               ms/PositiveInt
   type                   [:maybe supported-action-type]
   description            [:maybe :string]
   parameters             [:maybe [:sequential map?]]
   parameter_mappings     [:maybe map?]
   visualization_settings [:maybe map?]
   kind                   [:maybe implicit-action-kind]
   database_id            [:maybe ms/PositiveInt]
   dataset_query          [:maybe map?]
   template               [:maybe http-action-template]
   response_handle        [:maybe json-query-schema]
   error_handle           [:maybe json-query-schema]}
  (when (and (nil? database_id)
             (= "query" type))
    (throw (ex-info (tru "Must provide a database_id for query actions")
                    {:type        type
                     :status-code 400})))
  (let [model (api/write-check Card model_id)]
    (when (and (= "implicit" type)
               (not (card/model-supports-implicit-actions? model)))
      (throw (ex-info (tru "Implicit actions are not supported for models with clauses.")
                      {:status-code 400})))
    (doseq [db-id (cond-> [(:database_id model)] database_id (conj database_id))]
      (actions/check-actions-enabled-for-database!
       (t2/select-one Database :id db-id))))
  (let [action-id (action/insert! (assoc action :creator_id api/*current-user-id*))]
    (snowplow/track-event! ::snowplow/action-created api/*current-user-id* {:type           type
                                                                            :action_id      action-id
                                                                            :num_parameters (count parameters)})
    (if action-id
      (action/select-action :id action-id)
      ;; t2/insert! does not return a value when used with h2
      ;; so we return the most recently updated http action.
      (last (action/select-actions nil :type type)))))

(api/defendpoint PUT 
  [id :as {action :body}]
  {id     ms/PositiveInt
   action [:map
           [:archived               {:optional true} [:maybe :boolean]]
           [:database_id            {:optional true} [:maybe ms/PositiveInt]]
           [:dataset_query          {:optional true} [:maybe :map]]
           [:description            {:optional true} [:maybe :string]]
           [:error_handle           {:optional true} [:maybe json-query-schema]]
           [:kind                   {:optional true} [:maybe implicit-action-kind]]
           [:model_id               {:optional true} [:maybe ms/PositiveInt]]
           [:name                   {:optional true} [:maybe :string]]
           [:parameter_mappings     {:optional true} [:maybe :map]]
           [:parameters             {:optional true} [:maybe [:sequential :map]]]
           [:response_handle        {:optional true} [:maybe json-query-schema]]
           [:template               {:optional true} [:maybe http-action-template]]
           [:type                   {:optional true} [:maybe supported-action-type]]
           [:visualization_settings {:optional true} [:maybe :map]]]}
  (actions/check-actions-enabled! id)
  (let [existing-action (api/write-check Action id)]
    (action/update! (assoc action :id id) existing-action))
  (let [{:keys [parameters type] :as action} (action/select-action :id id)]
    (snowplow/track-event! ::snowplow/action-updated api/*current-user-id* {:type           type
                                                                            :action_id      id
                                                                            :num_parameters (count parameters)})
    action))

(api/defendpoint POST 
  "Generate publicly-accessible links for this Action. Returns UUID to be used in public links. (If this
  Action has already been shared, it will return the existing public link rather than creating a new one.) Public
  sharing must be enabled."
  [id]
  {id ms/PositiveInt}
  (api/check-superuser)
  (validation/check-public-sharing-enabled)
  (let [action (api/read-check Action id :archived false)]
    (actions/check-actions-enabled! action)
    {:uuid (or (:public_uuid action)
               (u/prog1 (str (random-uuid))
                 (t2/update! Action id
                             {:public_uuid <>
                              :made_public_by_id api/*current-user-id*})))}))

(api/defendpoint DELETE 
  "Delete the publicly-accessible link to this Dashboard."
  [id]
  {id ms/PositiveInt}
  ;; check the /application/setting permission, not superuser because removing a public link is possible from /admin/settings
  (validation/check-has-application-permission :setting)
  (validation/check-public-sharing-enabled)
  (api/check-exists? Action :id id, :public_uuid [:not= nil], :archived false)
  (actions/check-actions-enabled! id)
  (t2/update! Action id {:public_uuid nil, :made_public_by_id nil})
  {:status 204, :body nil})

(api/defendpoint GET 
  "Fetches the values for filling in execution parameters. Pass PK parameters and values to select."
  [action-id parameters]
  {action-id  ms/PositiveInt
   parameters ms/JSONString}
  (actions/check-actions-enabled! action-id)
  (-> (action/select-action :id action-id :archived false)
      api/read-check
      (actions.execution/fetch-values (json/parse-string parameters))))

(api/defendpoint POST 
  "Execute the Action.
   `parameters` should be the mapped dashboard parameters with values."
  [id :as {{:keys [parameters], :as _body} :body}]
  {id         ms/PositiveInt
   parameters [:maybe [:map-of :keyword any?]]}
  (let [{:keys [type] :as action} (api/check-404 (action/select-action :id id :archived false))]
    (snowplow/track-event! ::snowplow/action-executed api/*current-user-id* {:source    :model_detail
                                                                             :type      type
                                                                             :action_id id})
    (actions.execution/execute-action! action (update-keys parameters name))))

(api/define-routes)

(ns metabase.api.activity
  (:require
   [clojure.string :as str]
   [compojure.core :refer [GET]]
   [medley.core :as m]
   [metabase.api.common :as api :refer [*current-user-id* define-routes]]
   [metabase.db.util :as mdb.u]
   [metabase.models.card :refer [Card]]
   [metabase.models.dashboard :refer [Dashboard]]
   [metabase.models.interface :as mi]
   [metabase.models.query-execution :refer [QueryExecution]]
   [metabase.models.recent-views :as recent-views]
   [metabase.models.table :refer [Table]]
   [metabase.models.view-log :refer [ViewLog]]
   [metabase.util.honey-sql-2 :as h2x]
   [toucan2.core :as t2]))

(defn- models-query
  [model ids]
  (t2/select
   (case model
     "card"      [Card
                  :id :name :collection_id :description :display
                  :dataset_query :dataset :archived
                  :collection.authority_level]
     "dashboard" [Dashboard
                  :id :name :collection_id :description
                  :archived
                  :collection.authority_level]
     "table"     [Table
                  :id :name :db_id
                  :display_name :initial_sync_status
                  :visibility_type])
   (let [model-symb (symbol (str/capitalize model))
         self-qualify #(mdb.u/qualify model-symb %)]
     (cond-> {:where [:in (self-qualify :id) ids]}
       (not= model "table")
       (merge {:left-join [:collection [:= :collection.id (self-qualify :collection_id)]]})))))

(defn- select-items! [model ids]
  (when (seq ids)
    (for [model (t2/hydrate (models-query model ids) :moderation_reviews)
          :let [reviews (:moderation_reviews model)
                status  (->> reviews
                             (filter :most_recent)
                             first
                             :status)]]
      (assoc model :moderated_status status))))

(defn- models-for-views
  [views]
  (into {} (map (fn [[model models]]
                  [model (->> models
                              (map :model_id)
                              (select-items! model)
                              (m/index-by :id))]))
        (group-by :model views)))

(defn- views-and-runs
  [views-limit card-runs-limit all-users?]
  ;; TODO update to use RecentViews instead of ViewLog
  (let [dashboard-and-table-views (t2/select [ViewLog
                                              [[:min :view_log.user_id] :user_id]
                                              :model
                                              :model_id
                                              [:%count.* :cnt]
                                              [:%max.timestamp :max_ts]]
                                             {:group-by  [:model :model_id]
                                              :where     [:and
                                                          (when-not all-users? [:= (mdb.u/qualify ViewLog :user_id) *current-user-id*])
                                                          [:in :model #{"dashboard" "table"}]
                                                          [:= :bm.id nil]]
                                              :order-by  [[:max_ts :desc] [:model :desc]]
                                              :limit     views-limit
                                              :left-join [[:dashboard_bookmark :bm]
                                                          [:and
                                                           [:= :model "dashboard"]
                                                           [:= :bm.user_id *current-user-id*]
                                                           [:= :model_id :bm.dashboard_id]]]})
        card-runs                 (->> (t2/select [QueryExecution
                                                   [:%min.executor_id :user_id]
                                                   [(mdb.u/qualify QueryExecution :card_id) :model_id]
                                                   [:%count.* :cnt]
                                                   [:%max.started_at :max_ts]]
                                                  {:group-by [(mdb.u/qualify QueryExecution :card_id) :context]
                                                   :where    [:and
                                                              (when-not all-users? [:= :executor_id *current-user-id*])
                                                              [:= :context (h2x/literal :question)]
                                                              [:= :bm.id nil]]
                                                   :order-by [[:max_ts :desc]]
                                                   :limit    card-runs-limit
                                                   :left-join [[:card_bookmark :bm]
                                                               [:and
                                                                [:= :bm.user_id *current-user-id*]
                                                                [:= (mdb.u/qualify QueryExecution :card_id) :bm.card_id]]]})
                                       (map #(dissoc % :row_count))
                                       (map #(assoc % :model "card")))]
    (->> (concat card-runs dashboard-and-table-views)
         (sort-by :max_ts)
         reverse)))

(def ^:private views-limit 8)
(def ^:private card-runs-limit 8)

(api/defendpoint GET 
  "Get a list of 5 things the current user has been viewing most recently."
  []
  (let [views            (recent-views/user-recent-views api/*current-user-id* 10)
        model->id->items (models-for-views views)]
    (->> (for [{:keys [model model_id] :as view-log} views
               :let
               [model-object (-> (get-in model->id->items [model model_id])
                                 (dissoc :dataset_query))]
               :when
               (and model-object
                    (mi/can-read? model-object)
                    ;; hidden tables, archived cards/dashboards
                    (not (or (:archived model-object)
                             (= (:visibility_type model-object) :hidden))))]
           (cond-> (assoc view-log :model_object model-object)
             (:dataset model-object) (assoc :model "dataset")))
         (take 5))))

(api/defendpoint GET 
  "Get the most recently viewed dashboard for the current user. Returns a 204 if the user has not viewed any dashboards
   in the last 24 hours."
  []
  (if-let [dashboard-id (recent-views/most-recently-viewed-dashboard-id api/*current-user-id*)]
    (let [dashboard (-> (t2/select-one Dashboard :id dashboard-id)
                        api/check-404
                        (t2/hydrate [:collection :is_personal]))]
      (if (mi/can-read? dashboard)
        dashboard
        api/generic-204-no-content))
    api/generic-204-no-content))

(defn- official?
  [{:keys [authority_level]}]
  (boolean
   (when authority_level
     (#{"official"} authority_level))))

(defn- verified?
  [{:keys [moderated_status]}]
  (= moderated_status "verified"))

(defn- score-items
  [items]
  (when (seq items)
    (let [n-items (count items)
          max-count (apply max (map :cnt items))]
      (for [[recency-pos {:keys [cnt model_object] :as item}] (zipmap (range) items)]
        (let [verified-wt 1
              official-wt 1
              recency-wt 2
              views-wt 4
              scores [;; cards and dashboards? can be 'verified' in enterprise
                      (if (verified? model_object) verified-wt 0)
                      ;; items may exist in an 'official' collection in enterprise
                      (if (official? model_object) official-wt 0)
                      ;; most recent item = 1 * recency-wt, least recent item of 10 items = 1/10 * recency-wt
                      (* (/ (- n-items recency-pos) n-items) recency-wt)
                      ;; item with highest count = 1 * views-wt, lowest = item-view-count / max-view-count * views-wt
                      ;; NOTE: the query implementation `views-and-runs` has an order-by clause using most recent timestamp
                      ;; this has an effect on the outcomes. Consider an item with a massively high viewcount but a last view by the user
                      ;; a long time ago. This may not even make it into the firs 10 items from the query, even though it might be worth showing
                      (* (/ cnt max-count) views-wt)]]
          (assoc item :score (double (reduce + scores))))))))

(def ^:private model-precedence ["dashboard" "card" "dataset" "table"])

(defn- order-items
  [items]
  (when (seq items)
      (let [groups (group-by :model items)]
        (mapcat #(get groups %) model-precedence))))

(api/defendpoint GET 
  "Get the list of 5 popular things for the current user. Query takes 8 and limits to 5 so that if it
  finds anything archived, deleted, etc it can usually still get 5."
  []
  ;; we can do a weighted score which incorporates:
  ;; total count -> higher = higher score
  ;; recently viewed -> more recent = higher score
  ;; official/verified -> yes = higher score
  (let [views (views-and-runs views-limit card-runs-limit true)
        model->id->items (models-for-views views)
        filtered-views (for [{:keys [model model_id] :as view-log} views
                             :let [model-object (-> (get-in model->id->items [model model_id])
                                                    (dissoc :dataset_query))]
                             :when (and model-object
                                        (mi/can-read? model-object)
                                        ;; hidden tables, archived cards/dashboards
                                        (not (or (:archived model-object)
                                                 (= (:visibility_type model-object) :hidden))))]
                         (cond-> (assoc view-log :model_object model-object)
                           (:dataset model-object) (assoc :model "dataset")))
        scored-views (score-items filtered-views)]
    (->> scored-views
         (sort-by :score)
         reverse
         order-items
         (take 5)
         (map #(dissoc % :score)))))

(define-routes)

(ns metabase.api.alert
  (:require
   [clojure.data :as data]
   [clojure.set :refer [difference]]
   [compojure.core :refer [DELETE GET POST PUT]]
   [medley.core :as m]
   [metabase.api.common :as api]
   [metabase.api.common.validation :as validation]
   [metabase.config :as config]
   [metabase.email :as email]
   [metabase.email.messages :as messages]
   [metabase.events :as events]
   [metabase.models.card :refer [Card]]
   [metabase.models.interface :as mi]
   [metabase.models.pulse :as pulse]
   [metabase.models.pulse-channel :refer [PulseChannel]]
   [metabase.models.pulse-channel-recipient :refer [PulseChannelRecipient]]
   [metabase.plugins.classloader :as classloader]
   [metabase.public-settings.premium-features :as premium-features]
   [metabase.util :as u]
   [metabase.util.i18n :refer [tru]]
   [metabase.util.malli.schema :as ms]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(when config/ee-available?
  (classloader/require 'metabase-enterprise.advanced-permissions.common))

(api/defendpoint GET 
  "Fetch alerts which the current user has created or will receive, or all alerts if the user is an admin.
  The optional `user_id` will return alerts created by the corresponding user, but is ignored for non-admin users."
  [archived user_id]
  {archived [:maybe ms/BooleanString]
   user_id  [:maybe ms/PositiveInt]}
  (let [user-id (if api/*is-superuser?*
                  user_id
                  api/*current-user-id*)]
    (as-> (pulse/retrieve-alerts {:archived? (Boolean/parseBoolean archived)
                                  :user-id   user-id}) <>
      (filter mi/can-read? <>)
      (t2/hydrate <> :can_write))))

(api/defendpoint GET 
  "Fetch an alert by ID"
  [id]
  {id ms/PositiveInt}
  (-> (api/read-check (pulse/retrieve-alert id))
      (t2/hydrate :can_write)))

(api/defendpoint GET 
  "Fetch all alerts for the given question (`Card`) id"
  [id archived]
  {id       [:maybe ms/PositiveInt]
   archived [:maybe ms/BooleanString]}
  (-> (if api/*is-superuser?*
        (pulse/retrieve-alerts-for-cards {:card-ids [id], :archived? (Boolean/parseBoolean archived)})
        (pulse/retrieve-user-alerts-for-card {:card-id id, :user-id api/*current-user-id*, :archived? (Boolean/parseBoolean archived)}))
      (t2/hydrate :can_write)))

(defn- only-alert-keys [request]
  (u/select-keys-when request
    :present [:alert_condition :alert_first_only :alert_above_goal :archived]))

(defn email-channel
  [alert]
  (m/find-first #(= :email (keyword (:channel_type %))) (:channels alert)))

(defn- slack-channel
  [alert]
  (m/find-first #(= :slack (keyword (:channel_type %))) (:channels alert)))

(defn- key-by [key-fn coll]
  (zipmap (map key-fn coll) coll))

(defn- notify-email-disabled! [alert recipients]
  (doseq [user recipients]
    (messages/send-admin-unsubscribed-alert-email! alert user @api/*current-user*)))

(defn- notify-email-enabled! [alert recipients]
  (doseq [user recipients]
    (messages/send-you-were-added-alert-email! alert user @api/*current-user*)))

(defn- notify-email-recipient-diffs! [old-alert old-recipients new-alert new-recipients]
  (let [old-ids->users (key-by :id old-recipients)
        new-ids->users (key-by :id new-recipients)
        [removed-ids added-ids _] (data/diff (set (keys old-ids->users))
                                             (set (keys new-ids->users)))]
    (doseq [old-id removed-ids
            :let [removed-user (get old-ids->users old-id)]]
      (messages/send-admin-unsubscribed-alert-email! old-alert removed-user @api/*current-user*))
    (doseq [new-id added-ids
            :let [added-user (get new-ids->users new-id)]]
      (messages/send-you-were-added-alert-email! new-alert added-user @api/*current-user*))))

(defn- notify-recipient-changes!
  [old-alert updated-alert]
  (let [{old-recipients :recipients, old-enabled :enabled} (email-channel old-alert)
        {new-recipients :recipients, new-enabled :enabled} (email-channel updated-alert)]
    (cond
      ;; Did email notifications just get disabled?
      (and old-enabled (not new-enabled))
      (notify-email-disabled! old-alert old-recipients)
      ;; Did a disabled email notifications just get re-enabled?
      (and (not old-enabled) new-enabled)
      (notify-email-enabled! updated-alert new-recipients)
      ;; No need to notify recipients if emails are disabled
      new-enabled
      (notify-email-recipient-diffs! old-alert old-recipients updated-alert new-recipients))))

(defn- collect-alert-recipients [alert]
  (set (:recipients (email-channel alert))))

(defn- non-creator-recipients [{{creator-id :id} :creator :as alert}]
 (remove #(= creator-id (:id %)) (collect-alert-recipients alert)))

(defn- notify-new-alert-created! [alert]
  (when (email/email-configured?)
    (messages/send-new-alert-email! alert)
    (doseq [recipient (non-creator-recipients alert)]
      (messages/send-you-were-added-alert-email! alert recipient @api/*current-user*))))

(defn- maybe-include-csv [card alert-condition]
  (if (= "rows" alert-condition)
    (assoc card :include_csv true)
    card))

(api/defendpoint POST 
  "Create a new Alert."
  [:as {{:keys [alert_condition card channels alert_first_only alert_above_goal]
         :as new-alert-request-body} :body}]
  {alert_condition  pulse/AlertConditions
   alert_first_only :boolean
   alert_above_goal [:maybe :boolean]
   card             pulse/CardRef
   channels         [:+ :map]}
  (validation/check-has-application-permission :subscription false)
  ;; To create an Alert you need read perms for its Card
  (api/read-check Card (u/the-id card))
  ;; ok, now create the Alert
  (let [alert-card (-> card (maybe-include-csv alert_condition) pulse/card->ref)
        new-alert  (api/check-500
                    (-> new-alert-request-body
                        only-alert-keys
                        (pulse/create-alert! api/*current-user-id* alert-card channels)))]
   (events/publish-event! :event/alert-create {:object new-alert :user-id api/*current-user-id*})
   (notify-new-alert-created! new-alert)
    ;; return our new Alert
   new-alert))

(defn- notify-on-archive-if-needed!
  [alert]
  (when (email/email-configured?)
    (doseq [recipient (collect-alert-recipients alert)]
      (messages/send-admin-unsubscribed-alert-email! alert recipient @api/*current-user*))))

(api/defendpoint PUT 
  "Update a `Alert` with ID."
  [id :as {{:keys [alert_condition alert_first_only alert_above_goal card channels archived]
            :as alert-updates} :body}]
  {id               ms/PositiveInt
   alert_condition  [:maybe pulse/AlertConditions]
   alert_first_only [:maybe :boolean]
   alert_above_goal [:maybe :boolean]
   card             [:maybe pulse/CardRef]
   channels         [:maybe [:+ [:map]]]
   archived         [:maybe :boolean]}
  (try
   (validation/check-has-application-permission :monitoring)
   (catch clojure.lang.ExceptionInfo _e
     (validation/check-has-application-permission :subscription false)))
  ;; fetch the existing Alert in the DB
  (let [alert-before-update                   (api/check-404 (pulse/retrieve-alert id))
        current-user-has-application-permissions? (and (premium-features/enable-advanced-permissions?)
                                                   (resolve 'metabase-enterprise.advanced-permissions.common/current-user-has-application-permissions?))
        has-subscription-perms?               (and current-user-has-application-permissions?
                                                   (current-user-has-application-permissions? :subscription))
        has-monitoring-permissions?           (and current-user-has-application-permissions?
                                                   (current-user-has-application-permissions? :monitoring))]
    (assert (:card alert-before-update)
            (tru "Invalid Alert: Alert does not have a Card associated with it"))
    ;; check permissions as needed.
    ;; Check permissions to update existing Card
    (api/read-check Card (u/the-id (:card alert-before-update)))
    ;; if trying to change the card, check perms for that as well
    (when card
      (api/write-check Card (u/the-id card)))
    (when-not (or api/*is-superuser?*
                  has-monitoring-permissions?
                  has-subscription-perms?)
      (api/check (= (-> alert-before-update :creator :id) api/*current-user-id*)
                 [403 (tru "Non-admin users without monitoring or subscription permissions are only allowed to update alerts that they created")])
      (api/check (or (not (contains? alert-updates :channels))
                     (and (= 1 (count channels))
                          ;; Non-admin alerts can only include the creator as a recipient
                          (= [api/*current-user-id*]
                             (map :id (:recipients (email-channel alert-updates))))))
                 [403 (tru "Non-admin users without monitoring or subscription permissions are not allowed to modify the channels for an alert")]))
    ;; only admin or users with subscription permissions can add recipients
    (let [to-add-recipients (difference (set (map :id (:recipients (email-channel alert-updates))))
                                        (set (map :id (:recipients (email-channel alert-before-update)))))]
      (api/check (or api/*is-superuser?*
                     has-subscription-perms?
                     (empty? to-add-recipients))
                 [403 (tru "Non-admin users without subscription permissions are not allowed to add recipients")]))
    ;; now update the Alert
    (let [updated-alert (pulse/update-alert!
                         (merge
                          (assoc (only-alert-keys alert-updates)
                                 :id id)
                          (when card
                            {:card (pulse/card->ref card)})
                          (when (contains? alert-updates :channels)
                            {:channels channels})
                          ;; automatically archive alert if it now has no recipients
                          (when (and (contains? alert-updates :channels)
                                     (not (seq (:recipients (email-channel alert-updates))))
                                     (not (slack-channel alert-updates)))
                            {:archived true})))]
      ;; Only admins or users has subscription or monitoring perms
      ;; can update recipients or explicitly archive an alert
      (when (and (or api/*is-superuser?*
                     has-subscription-perms?
                     has-monitoring-permissions?)
                 (email/email-configured?))
        (if archived
          (notify-on-archive-if-needed! updated-alert)
          (notify-recipient-changes! alert-before-update updated-alert)))
      ;; Finally, return the updated Alert
      updated-alert)))

(api/defendpoint DELETE 
  "For users to unsubscribe themselves from the given alert."
  [id]
  {id ms/PositiveInt}
  (validation/check-has-application-permission :subscription false)
  (let [alert (pulse/retrieve-alert id)]
    (api/read-check alert)
    (api/let-404 [alert-id (u/the-id alert)
                  pc-id    (t2/select-one-pk PulseChannel :pulse_id alert-id :channel_type "email")
                  pcr-id   (t2/select-one-pk PulseChannelRecipient :pulse_channel_id pc-id :user_id api/*current-user-id*)]
                 (t2/delete! PulseChannelRecipient :id pcr-id))
    ;; Send emails letting people know they have been unsubscribed
    (let [user @api/*current-user*]
      (when (email/email-configured?)
        (messages/send-you-unsubscribed-alert-email! alert user))
      (events/publish-event! :event/alert-unsubscribe {:object {:email (:email user)}
                                                       :user-id api/*current-user-id*}))
    ;; finally, return a 204 No Content
    api/generic-204-no-content))

(api/define-routes)

(ns metabase.api.api-key
  (:require
   [compojure.core :refer [POST GET PUT DELETE]]
   [metabase.api.common :as api]
   [metabase.events :as events]
   [metabase.models.api-key :as api-key]
   [metabase.models.permissions-group :as perms-group]
   [metabase.models.user :as user]
   [metabase.util :as u]
   [metabase.util.i18n :refer [tru]]
   [metabase.util.malli.schema :as ms]
   [metabase.util.secret :as u.secret]
   [toucan2.core :as t2]))

(defn- maybe-expose-key [api-key]
  (if (contains? api-key :unmasked_key)
    (update api-key :unmasked_key u.secret/expose)
    api-key))

(defn- present-api-key
  [api-key]
  (-> api-key
      (t2/hydrate :group :updated_by)
      (select-keys [:created_at
                    :updated_at
                    :updated_by
                    :id
                    :group
                    :unmasked_key
                    :name
                    :masked_key])
      (maybe-expose-key)
      (update :updated_by #(select-keys % [:common_name :id]))))

(defn- key-with-unique-prefix []
  (u/auto-retry 5
   (let [api-key (api-key/generate-key)
         prefix (api-key/prefix (u.secret/expose api-key))]
     ;; we could make this more efficient by generating 5 API keys up front and doing one select to remove any
     ;; duplicates. But a duplicate should be rare enough to just do multiple queries for now.
     (if-not (t2/exists? :model/ApiKey :key_prefix prefix)
       api-key
       (throw (ex-info (tru "could not generate key with unique prefix") {}))))))

(defn- with-updated-by [api-key]
  (assoc api-key :updated_by_id api/*current-user-id*))

(defn- with-creator [api-key]
  (assoc api-key :creator_id api/*current-user-id*))

(api/defendpoint POST 
  "Create a new API key (and an associated `User`) with the provided name and group ID."
  [:as {{:keys [group_id name] :as _body} :body}]
  {group_id ms/PositiveInt
   name     ms/NonBlankString}
  (api/check-superuser)
  (api/checkp (not (t2/exists? :model/ApiKey :name name))
    "name" "An API key with this name already exists.")
  (let [unhashed-key (key-with-unique-prefix)
        email        (format "api-key-user-%s@api-key.invalid" (u/slugify name))]
    (t2/with-transaction [_conn]
      (let [user (first
                  (t2/insert-returning-instances! :model/User
                                                  {:email      email
                                                   :first_name name
                                                   :last_name  ""
                                                   :type       :api-key
                                                   :password (str (random-uuid))}))]
        (user/set-permissions-groups! user [(perms-group/all-users) group_id])
        (let [api-key (-> (t2/insert-returning-instance! :model/ApiKey
                                                         (-> {:user_id       (u/the-id user)
                                                              :name          name
                                                              :unhashed_key  unhashed-key}
                                                             with-creator
                                                             with-updated-by))
                          (t2/hydrate :group :updated_by))]
          (events/publish-event! :event/api-key-create
                                 {:object  api-key
                                  :user-id api/*current-user-id*})
          (present-api-key (assoc api-key :unmasked_key unhashed-key)))))))

(api/defendpoint GET 
  "Get the count of API keys in the DB"
  [:as _body]
  (api/check-superuser)
  (t2/count :model/ApiKey))

(api/defendpoint PUT 
  "Update an API key by changing its group and/or its name"
  [id :as {{:keys [group_id name] :as _body} :body}]
  {id       ms/PositiveInt
   group_id [:maybe ms/PositiveInt]
   name     [:maybe ms/NonBlankString]}
  (api/check-superuser)
  (let [api-key-before (-> (t2/select-one :model/ApiKey :id id)
                           ;; hydrate the group_name for audit logging
                           (t2/hydrate :group)
                           (api/check-404))]
    (t2/with-transaction [_conn]
      (when group_id
        (let [user (-> api-key-before (t2/hydrate :user) :user)]
          (user/set-permissions-groups! user [(perms-group/all-users) {:id group_id}])))
      (when name
        ;; A bit of a pain to keep these in sync, but oh well.
        (t2/update! :model/User (:user_id api-key-before) {:first_name name
                                                           :last_name ""})
        (t2/update! :model/ApiKey id (with-updated-by {:name name}))))
    (let [updated-api-key (-> (t2/select-one :model/ApiKey :id id)
                              (t2/hydrate :group :updated_by))]
      (events/publish-event! :event/api-key-update {:object          updated-api-key
                                                    :previous-object api-key-before
                                                    :user-id         api/*current-user-id*})
      (present-api-key updated-api-key))))

(api/defendpoint PUT 
  "Regenerate an API Key"
  [id]
  {id ms/PositiveInt}
  (api/check-superuser)
  (let [api-key-before (-> (t2/select-one :model/ApiKey id)
                           (t2/hydrate :group)
                           (api/check-404))
        unhashed-key (key-with-unique-prefix)
        api-key-after (assoc api-key-before
                             :unhashed_key unhashed-key
                             :key_prefix (api-key/prefix (u.secret/expose unhashed-key)))]
    (t2/update! :model/ApiKey :id id (with-updated-by
                                       (select-keys api-key-after [:unhashed_key])))
    (events/publish-event! :event/api-key-regenerate
                           {:object api-key-after
                            :previous-object api-key-before
                            :user-id api/*current-user-id*})
    (present-api-key (assoc api-key-after
                            :unmasked_key unhashed-key
                            :masked_key (api-key/mask unhashed-key)))))

(api/defendpoint GET 
  "Get a list of API keys. Non-paginated."
  []
  (api/check-superuser)
  (let [api-keys (t2/hydrate (t2/select :model/ApiKey) :group :updated_by)]
    (map present-api-key api-keys)))

(api/defendpoint DELETE 
  "Delete an ApiKey"
  [id]
  {id ms/PositiveInt}
  (api/check-superuser)
  (let [api-key (-> (t2/select-one :model/ApiKey id)
                    (t2/hydrate :group)
                    (api/check-404))]
    (t2/with-transaction [_tx]
      (t2/delete! :model/ApiKey id)
      (t2/update! :model/User (:user_id api-key) {:is_active false}))
    (events/publish-event! :event/api-key-delete
                           {:object api-key
                            :user-id api/*current-user-id*})
    api/generic-204-no-content))

(api/define-routes)

(ns metabase.api.automagic-dashboards
  (:require
   [buddy.core.codecs :as codecs]
   [cheshire.core :as json]
   [compojure.core :refer [GET]]
   [metabase.api.common :as api]
   [metabase.automagic-dashboards.comparison :refer [comparison-dashboard]]
   [metabase.automagic-dashboards.core :as magic
    :refer [automagic-analysis candidate-tables]]
   [metabase.automagic-dashboards.dashboard-templates :as dashboard-templates]
   [metabase.models.card :refer [Card]]
   [metabase.models.collection :refer [Collection]]
   [metabase.models.database :refer [Database]]
   [metabase.models.field :refer [Field]]
   [metabase.models.metric :refer [Metric]]
   [metabase.models.model-index :refer [ModelIndex ModelIndexValue]]
   [metabase.models.permissions :as perms]
   [metabase.models.query :as query]
   [metabase.models.query.permissions :as query-perms]
   [metabase.models.segment :refer [Segment]]
   [metabase.models.table :refer [Table]]
   [metabase.transforms.dashboard :as transform.dashboard]
   [metabase.transforms.materialize :as tf.materialize]
   [metabase.util.i18n :refer [deferred-tru]]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [ring.util.codec :as codec]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(def ^:private Show
  (mu/with-api-error-message
    [:maybe [:or [:enum "all"] nat-int?]]
    (deferred-tru "invalid show value")))

(def ^:private Prefix
  (mu/with-api-error-message
    [:fn (fn [prefix]
           (some #(not-empty (dashboard-templates/get-dashboard-templates [% prefix])) ["table" "metric" "field"]))]
    (deferred-tru "invalid value for prefix")))

(def ^:private DashboardTemplate
  (mu/with-api-error-message
    [:fn (fn [dashboard-template]
           (some (fn [toplevel]
                   (some (comp dashboard-templates/get-dashboard-template
                               (fn [prefix]
                                 [toplevel prefix dashboard-template])
                               :dashboard-template-name)
                         (dashboard-templates/get-dashboard-templates [toplevel])))
                 ["table" "metric" "field"]))]
    (deferred-tru "invalid value for dashboard template name")))

(def ^:private ^{:arglists '([s])} decode-base64-json
  (comp #(json/decode % keyword) codecs/bytes->str codec/base64-decode))

(def ^:private Base64EncodedJSON
  (mu/with-api-error-message
    [:fn decode-base64-json]
    (deferred-tru "value couldn''t be parsed as base64 encoded JSON")))

(api/defendpoint GET 
  "Return a list of candidates for automagic dashboards orderd by interestingness."
  [id]
  {id ms/PositiveInt}
  (-> (t2/select-one Database :id id)
      api/read-check
      candidate-tables))

(defn- adhoc-query-read-check
  [query]
  (api/check-403 (perms/set-has-partial-permissions-for-set?
                   @api/*current-user-permissions-set*
                   (query-perms/perms-set (:dataset_query query), :throw-exceptions? true)))
  query)

(defn- ensure-int
  [x]
  (if (string? x)
    (Integer/parseInt x)
    x))

(defmulti ^:private ->entity
  {:arglists '([entity-type s])}
  (fn [entity-type _s]
    (keyword entity-type)))

(defmethod ->entity :table
  [_entity-type table-id-str]
  ;; table-id can also be a source query reference like `card__1` so in that case we should pull the ID out and use the
  ;; `:question` method instead
  (if-let [[_ card-id-str] (when (string? table-id-str)
                             (re-matches #"^card__(\d+$)" table-id-str))]
    (->entity :question card-id-str)
    (api/read-check (t2/select-one Table :id (ensure-int table-id-str)))))

(defmethod ->entity :segment
  [_entity-type segment-id-str]
  (api/read-check (t2/select-one Segment :id (ensure-int segment-id-str))))

(defmethod ->entity :model
  [_entity-type card-id-str]
  (api/read-check (t2/select-one Card
                                 :id (ensure-int card-id-str)
                                 :dataset true)))

(defmethod ->entity :question
  [_entity-type card-id-str]
  (api/read-check (t2/select-one Card :id (ensure-int card-id-str))))

(defmethod ->entity :adhoc
  [_entity-type encoded-query]
  (adhoc-query-read-check (query/adhoc-query (decode-base64-json encoded-query))))

(defmethod ->entity :metric
  [_entity-type metric-id-str]
  (api/read-check (t2/select-one Metric :id (ensure-int metric-id-str))))

(defmethod ->entity :field
  [_entity-type field-id-str]
  (api/read-check (t2/select-one Field :id (ensure-int field-id-str))))

(defmethod ->entity :transform
  [_entity-type transform-name]
  (api/read-check (t2/select-one Collection :id (tf.materialize/get-collection transform-name)))
  transform-name)

(def ^:private entities
  (map name (keys (methods ->entity))))

(def ^:private Entity
  (mu/with-api-error-message
    (into [:enum] entities)
    (deferred-tru "Invalid entity type")))

(def ^:private ComparisonEntity
  (mu/with-api-error-message
    [:enum "segment" "adhoc" "table"]
    (deferred-tru "Invalid comparison entity type. Can only be one of \"table\", \"segment\", or \"adhoc\)))

(defn- coerce-show
  [show]
  (cond-> show (= "all" show) keyword))

(api/defendpoint GET 
  "Return an automagic dashboard for entity `entity` with id `id`."
  [entity entity-id-or-query show]
  {show   [:maybe [:or [:= "all"] nat-int?]]
   entity (mu/with-api-error-message
            (into [:enum] entities)
            (deferred-tru "Invalid entity type"))}
  (if (= entity "transform")
    (transform.dashboard/dashboard (->entity entity entity-id-or-query))
    (-> (->entity entity entity-id-or-query)
        (automagic-analysis {:show (coerce-show show)}))))

(defn linked-entities
  [{{field-ref :pk_ref} :model-index {rsmd :result_metadata} :model}]
  (when-let [field-id (:id (some #(when ((comp #{field-ref} :field_ref) %) %) rsmd))]
    (map
     (fn [{:keys [table_id id]}]
       {:linked-table-id table_id
        :linked-field-id id})
     (t2/select 'Field :fk_target_field_id field-id))))

(defn- add-source-model-link
  [{model-name :name model-id :id} cards]
  (let [max-width (->> (map (fn [{:keys [col size_x]}] (+ col size_x)) cards)
                       (into [4])
                       (apply max))]
    (cons
     {:id                     (gensym)
      :size_x                 max-width
      :size_y                 1
      :row                    0
      :col                    0
      :visualization_settings {:virtual_card {:display  "link"
                                              :archived false},
                               :link         {:entity {:id          model-id
                                                       :name        model-name
                                                       :model       "dataset"
                                                       :display     "table"
                                                       :description nil}}}}
     cards)))

(defn- create-linked-dashboard
  [{{indexed-entity-name :name :keys [model_pk]} :model-index-value
    {model-name :name :as model}                 :model
    :keys                                        [linked-tables]}]
  (if (seq linked-tables)
    (let [child-dashboards (map (fn [{:keys [linked-table-id linked-field-id]}]
                                  (let [table (t2/select-one Table :id linked-table-id)]
                                    (magic/automagic-analysis
                                     table
                                     {:show         :all
                                      :query-filter [:= [:field linked-field-id nil] model_pk]})))
                                linked-tables)
          seed-dashboard   (-> (first child-dashboards)
                               (merge
                                {:name         (format "Here's a look at \"%s\" from \"%s\"" indexed-entity-name model-name)
                                 :description  (format "A dashboard focusing on information linked to %s" indexed-entity-name)
                                 :parameters   []
                                 :param_fields {}})
                               (dissoc :transient_name
                                       :transient_filters))]
      (if (second child-dashboards)
        (->> child-dashboards
             (map-indexed (fn [idx {tab-name :name tab-cards :dashcards}]
                            ;; id starts at 0. want our temporary ids to start at -1, -2, ...
                            (let [tab-id (dec (- idx))]
                              {:tab {:id       tab-id
                                     :name     tab-name
                                     :position idx}
                               :dash-cards
                               (map (fn [dc]
                                      (assoc dc :dashboard_tab_id tab-id))
                                    (add-source-model-link model tab-cards))})))
             (reduce (fn [dashboard {:keys [tab dash-cards]}]
                       (-> dashboard
                           (update :dashcards into dash-cards)
                           (update :tabs conj tab)))
                     (merge
                      seed-dashboard
                      {:dashcards []
                       :tabs      []})))
        (update seed-dashboard
                :dashcards (fn [cards] (add-source-model-link model cards)))))
    {:name      (format "Here's a look at \"%s\" from \"%s\"" indexed-entity-name model-name)
     :dashcards (add-source-model-link
                     model
                     [{:row                    0
                       :col                    0
                       :size_x                 18
                       :size_y                 2
                       :visualization_settings {:text                "# Unfortunately, there's not much else to show right now..."
                                                :virtual_card        {:display :text}
                                                :dashcard.background false
                                                :text.align_vertical :bottom}}])}))

(api/defendpoint GET 
  "Return an automagic dashboard for an entity detail specified by `entity`
  with id `id` and a primary key of `indexed-value`."
  [model-index-id pk-id]
  {model-index-id :int
   pk-id          :int}
  (api/let-404 [model-index (t2/select-one ModelIndex model-index-id)
                model (t2/select-one Card (:model_id model-index))
                model-index-value (t2/select-one ModelIndexValue
                                                 :model_index_id model-index-id
                                                 :model_pk pk-id)]
               ;; `->entity` does a read check on the model but this is here as well to be extra sure.
    (api/read-check Card (:model_id model-index))
    (let [linked (linked-entities {:model             model
                                   :model-index       model-index
                                   :model-index-value model-index-value})]
      (create-linked-dashboard {:model             model
                                :linked-tables     linked
                                :model-index       model-index
                                :model-index-value model-index-value}))))

(api/defendpoint GET 
  "Return an automagic dashboard for entity `entity` with id `id` using dashboard-template `dashboard-template`."
  [entity entity-id-or-query prefix dashboard-template show]
  {entity Entity
   show   Show
   prefix Prefix
   dashboard-template   DashboardTemplate}
  (-> (->entity entity entity-id-or-query)
      (automagic-analysis {:show (coerce-show show)
                           :dashboard-template ["table" prefix dashboard-template]})))

(api/defendpoint GET 
  "Return an automagic dashboard analyzing cell in  automagic dashboard for entity `entity`
   defined by
   query `cell-query`."
  [entity entity-id-or-query cell-query show]
  {entity     Entity
   show       Show
   cell-query Base64EncodedJSON}
  (-> (->entity entity entity-id-or-query)
      (automagic-analysis {:show       (coerce-show show)
                           :cell-query (decode-base64-json cell-query)})))

(api/defendpoint GET 
  "Return an automagic dashboard analyzing cell in question  with id `id` defined by
   query `cell-query` using dashboard-template `dashboard-template`."
  [entity entity-id-or-query cell-query prefix dashboard-template show]
  {entity     Entity
   show       Show
   prefix     Prefix
   dashboard-template       DashboardTemplate
   cell-query Base64EncodedJSON}
  (-> (->entity entity entity-id-or-query)
      (automagic-analysis {:show       (coerce-show show)
                           :dashboard-template       ["table" prefix dashboard-template]
                           :cell-query (decode-base64-json cell-query)})))

(api/defendpoint GET 
  "Return an automagic comparison dashboard for entity `entity` with id `id` compared with entity
   `comparison-entity` with id `comparison-entity-id-or-query.`"
  [entity entity-id-or-query show comparison-entity comparison-entity-id-or-query]
  {show              Show
   entity            Entity
   comparison-entity ComparisonEntity}
  (let [left      (->entity entity entity-id-or-query)
        right     (->entity comparison-entity comparison-entity-id-or-query)
        dashboard (automagic-analysis left {:show         (coerce-show show)
                                            :query-filter nil
                                            :comparison?  true})]
    (comparison-dashboard dashboard left right {})))

(api/defendpoint GET 
  "Return an automagic comparison dashboard for entity `entity` with id `id` using dashboard-template `dashboard-template`;
   compared with entity `comparison-entity` with id `comparison-entity-id-or-query.`."
  [entity entity-id-or-query prefix dashboard-template show comparison-entity comparison-entity-id-or-query]
  {entity            Entity
   show              Show
   prefix            Prefix
   dashboard-template              DashboardTemplate
   comparison-entity ComparisonEntity}
  (let [left      (->entity entity entity-id-or-query)
        right     (->entity comparison-entity comparison-entity-id-or-query)
        dashboard (automagic-analysis left {:show         (coerce-show show)
                                            :dashboard-template         ["table" prefix dashboard-template]
                                            :query-filter nil
                                            :comparison?  true})]
    (comparison-dashboard dashboard left right {})))

(api/defendpoint GET 
  "Return an automagic comparison dashboard for cell in automagic dashboard for entity `entity`
   with id `id` defined by query `cell-query`; compared with entity `comparison-entity` with id
   `comparison-entity-id-or-query.`."
  [entity entity-id-or-query cell-query show comparison-entity comparison-entity-id-or-query]
  {entity            Entity
   show              Show
   cell-query        Base64EncodedJSON
   comparison-entity ComparisonEntity}
  (let [left      (->entity entity entity-id-or-query)
        right     (->entity comparison-entity comparison-entity-id-or-query)
        dashboard (automagic-analysis left {:show         (coerce-show show)
                                            :query-filter nil
                                            :comparison?  true})]
    (comparison-dashboard dashboard left right {:left {:cell-query (decode-base64-json cell-query)}})))

(api/defendpoint GET 
  "Return an automagic comparison dashboard for cell in automagic dashboard for entity `entity`
   with id `id` defined by query `cell-query` using dashboard-template `dashboard-template`; compared with entity
   `comparison-entity` with id `comparison-entity-id-or-query.`."
  [entity entity-id-or-query cell-query prefix dashboard-template show comparison-entity comparison-entity-id-or-query]
  {entity            Entity
   show              Show
   prefix            Prefix
   dashboard-template              DashboardTemplate
   cell-query        Base64EncodedJSON
   comparison-entity ComparisonEntity}
  (let [left      (->entity entity entity-id-or-query)
        right     (->entity comparison-entity comparison-entity-id-or-query)
        dashboard (automagic-analysis left {:show         (coerce-show show)
                                            :dashboard-template         ["table" prefix dashboard-template]
                                            :query-filter nil})]
    (comparison-dashboard dashboard left right {:left {:cell-query (decode-base64-json cell-query)}})))

(api/define-routes)

(ns metabase.api.bookmark
  (:require
   [compojure.core :refer [DELETE GET POST]]
   [metabase.api.common :as api]
   [metabase.models.bookmark
    :as bookmark
    :refer [CardBookmark CollectionBookmark DashboardBookmark]]
   [metabase.models.card :refer [Card]]
   [metabase.models.collection :refer [Collection]]
   [metabase.models.dashboard :refer [Dashboard]]
   [metabase.util.malli.schema :as ms]
   [toucan2.core :as t2]))

(def Models
  (into [:enum] ["card" "dashboard" "collection"]))

(def BookmarkOrderings
  [:sequential [:map
                [:type Models]
                [:item_id ms/PositiveInt]]])

(def ^:private lookup
  {"card"       [Card       CardBookmark       :card_id]
   "dashboard"  [Dashboard  DashboardBookmark  :dashboard_id]
   "collection" [Collection CollectionBookmark :collection_id]})

(api/defendpoint GET 
  "Fetch all bookmarks for the user"
  []
  ;; already sorted by created_at in query. Can optionally use user sort preferences here and not in the function
  ;; below
  (bookmark/bookmarks-for-user api/*current-user-id*))

(api/defendpoint POST 
  "Create a new bookmark for user."
  [model id]
  {model Models
   id    ms/PositiveInt}
  (let [[item-model bookmark-model item-key] (lookup model)]
    (api/read-check item-model id)
    (api/check (not (t2/exists? bookmark-model item-key id
                                :user_id api/*current-user-id*))
      [400 "Bookmark already exists"])
    (first (t2/insert-returning-instances! bookmark-model {item-key id :user_id api/*current-user-id*}))))

(api/defendpoint DELETE 
  "Delete a bookmark. Will delete a bookmark assigned to the user making the request by model and id."
  [model id]
  {model Models
   id    ms/PositiveInt}
  ;; todo: allow admins to include an optional user id to delete for so they can delete other's bookmarks.
  (let [[_ bookmark-model item-key] (lookup model)]
    (t2/delete! bookmark-model
                :user_id api/*current-user-id*
                item-key id)
    api/generic-204-no-content))

(api/defendpoint PUT 
  "Sets the order of bookmarks for user."
  [:as {{:keys [orderings]} :body}]
  {orderings BookmarkOrderings}
  (bookmark/save-ordering! api/*current-user-id* orderings)
  api/generic-204-no-content)

(api/define-routes)

(ns metabase.api.card
  (:require
   [cheshire.core :as json]
   [clojure.core.async :as a]
   [clojure.java.io :as io]
   [compojure.core :refer [DELETE GET POST PUT]]
   [medley.core :as m]
   [metabase.api.common :as api]
   [metabase.api.common.validation :as validation]
   [metabase.api.dataset :as api.dataset]
   [metabase.api.field :as api.field]
   [metabase.driver :as driver]
   [metabase.events :as events]
   [metabase.lib.types.isa :as lib.types.isa]
   [metabase.mbql.normalize :as mbql.normalize]
   [metabase.mbql.util :as mbql.u]
   [metabase.models
    :refer [Card CardBookmark Collection Database PersistedInfo Table]]
   [metabase.models.card :as card]
   [metabase.models.collection :as collection]
   [metabase.models.collection.root :as collection.root]
   [metabase.models.interface :as mi]
   [metabase.models.params :as params]
   [metabase.models.params.custom-values :as custom-values]
   [metabase.models.persisted-info :as persisted-info]
   [metabase.models.query :as query]
   [metabase.models.query.permissions :as query-perms]
   [metabase.models.revision.last-edit :as last-edit]
   [metabase.models.timeline :as timeline]
   [metabase.public-settings :as public-settings]
   [metabase.public-settings.premium-features :as premium-features]
   [metabase.query-processor.card :as qp.card]
   [metabase.query-processor.pivot :as qp.pivot]
   [metabase.related :as related]
   [metabase.server.middleware.offset-paging :as mw.offset-paging]
   [metabase.sync.analyze.query-results :as qr]
   [metabase.task.persist-refresh :as task.persist-refresh]
   [metabase.upload :as upload]
   [metabase.util :as u]
   [metabase.util.date-2 :as u.date]
   [metabase.util.i18n :refer [deferred-tru trs tru]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [steffan-westcott.clj-otel.api.trace.span :as span]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(defmulti ^:private cards-for-filter-option*
  {:arglists '([filter-option & args])}
  (fn [filter-option & _]
    (keyword filter-option)))

(defmethod cards-for-filter-option* :all
  [_]
  (t2/select Card, :archived false, {:order-by [[:%lower.name :asc]]}))

(defmethod cards-for-filter-option* :mine
  [_]
  (t2/select Card, :creator_id api/*current-user-id*, :archived false, {:order-by [[:%lower.name :asc]]}))

(defmethod cards-for-filter-option* :bookmarked
  [_]
  (let [cards (for [{{:keys [archived], :as card} :card} (t2/hydrate (t2/select [CardBookmark :card_id]
                                                                      :user_id api/*current-user-id*)
                                                                  :card)
                    :when                                 (not archived)]
                card)]
    (sort-by :name cards)))

(defmethod cards-for-filter-option* :database
  [_ database-id]
  (t2/select Card, :database_id database-id, :archived false, {:order-by [[:%lower.name :asc]]}))

(defmethod cards-for-filter-option* :table
  [_ table-id]
  (t2/select Card, :table_id table-id, :archived false, {:order-by [[:%lower.name :asc]]}))

(defmethod cards-for-filter-option* :archived
  [_]
  (t2/select Card, :archived true, {:order-by [[:%lower.name :asc]]}))

(defmethod cards-for-filter-option* :using_model
  [_filter-option model-id]
  (->> (t2/select Card {:select [:c.*]
                        :from [[:report_card :m]]
                        :join [[:report_card :c] [:and
                                                  [:= :c.database_id :m.database_id]
                                                  [:or
                                                   [:like :c.dataset_query (format "%%card__%s%%" model-id)]
                                                   [:like :c.dataset_query (format "%%#%s%%" model-id)]]]]
                        :where [:and [:= :m.id model-id] [:not :c.archived]]})
       ;; now check if model-id really occurs as a card ID
       (filter (fn [card] (some #{model-id} (-> card :dataset_query query/collect-card-ids))))))

(defn- cards-for-filter-option [filter-option model-id-or-nil]
  (-> (apply cards-for-filter-option* filter-option (when model-id-or-nil [model-id-or-nil]))
      (t2/hydrate :creator :collection)))

(def ^:private card-filter-options
  (map name (keys (methods cards-for-filter-option*))))

(api/defendpoint GET 
  "Get all the Cards. Option filter param `f` can be used to change the set of Cards that are returned; default is
  `all`, but other options include `mine`, `bookmarked`, `database`, `table`, `using_model` and `archived`. See
  corresponding implementation functions above for the specific behavior of each filterp option. :card_index:"
  [f model_id]
  {f        [:maybe (into [:enum] card-filter-options)]
   model_id [:maybe ms/PositiveInt]}
  (let [f (or (keyword f) :all)]
    (when (contains? #{:database :table :using_model} f)
      (api/checkp (integer? model_id) "model_id" (format "model_id is a required parameter when filter mode is '%s'"
                                                         (name f)))
      (case f
        :database    (api/read-check Database model_id)
        :table       (api/read-check Database (t2/select-one-fn :db_id Table, :id model_id))
        :using_model (api/read-check Card model_id)))
    (let [cards          (filter mi/can-read? (cards-for-filter-option f model_id))
          last-edit-info (:card (last-edit/fetch-last-edited-info {:card-ids (map :id cards)}))]
      (into []
            (map (fn [{:keys [id] :as card}]
                   (if-let [edit-info (get last-edit-info id)]
                     (assoc card :last-edit-info edit-info)
                     card)))
            cards))))

(defn hydrate-card-details
  [{card-id :id :as card}]
  (span/with-span!
    {:name       "hydrate-card-details"
     :attributes {:card/id card-id}}
    (-> card
        (t2/hydrate :creator
                    :dashboard_count
                    :can_write
                    :average_query_time
                    :last_query_start
                    :parameter_usage_count
                    [:collection :is_personal]
                    [:moderation_reviews :moderator_details])
        (cond->                                             ; card
          (:dataset card) (t2/hydrate :persisted)))))

(api/defendpoint GET 
  "Get `Card` with ID."
  [id ignore_view]
  {id ms/PositiveInt
   ignore_view [:maybe :boolean]}
  (let [raw-card (t2/select-one Card :id id)
        card (-> raw-card
                 api/read-check
                 hydrate-card-details
                 ;; Cal 2023-11-27: why is last-edit-info hydrated differently for GET vs PUT and POST
                 (last-edit/with-last-edit-info :card)
                 collection.root/hydrate-root-collection)]
    (u/prog1 card
      (when-not ignore_view
        (events/publish-event! :event/card-read {:object <> :user-id api/*current-user-id*})))))

(defn- card-columns-from-names
  [card names]
  (when-let [names (set names)]
    (filter #(names (:name %)) (:result_metadata card))))

(defn- cols->kebab-case
  [cols]
  (map #(update-keys % u/->kebab-case-en) cols))

(defn- area-bar-line-series-are-compatible?
  [first-card second-card]
  (and (#{:area :line :bar} (:display second-card))
       (let [initial-dimensions (cols->kebab-case
                                  (card-columns-from-names
                                    first-card
                                    (get-in first-card [:visualization_settings :graph.dimensions])))
             new-dimensions     (cols->kebab-case
                                  (card-columns-from-names
                                    second-card
                                    (get-in second-card [:visualization_settings :graph.dimensions])))
             new-metrics        (cols->kebab-case
                                  (card-columns-from-names
                                    second-card
                                    (get-in second-card [:visualization_settings :graph.metrics])))]
         (cond
           ;; must have at least one dimension and one metric
           (or (zero? (count new-dimensions))
               (zero? (count new-metrics)))
           false
           ;; all metrics must be numeric
           (not (every? lib.types.isa/numeric? new-metrics))
           false
           ;; both or neither primary dimension must be dates
           (not= (lib.types.isa/temporal? (first initial-dimensions))
                 (lib.types.isa/temporal? (first new-dimensions)))
           false
           ;; both or neither primary dimension must be numeric
           ;; a timestamp field is both date and number so don't enforce the condition if both fields are dates; see #2811
           (and (not= (lib.types.isa/numeric? (first initial-dimensions))
                      (lib.types.isa/numeric? (first new-dimensions)))
                (not (and
                      (lib.types.isa/temporal? (first initial-dimensions))
                      (lib.types.isa/temporal? (first new-dimensions)))))
           false
           :else true))))

(defmulti series-are-compatible?
  (fn [card _second-card]
   (:display card)))

(defmethod series-are-compatible? :area
  [first-card second-card]
  (area-bar-line-series-are-compatible? first-card second-card))

(defmethod series-are-compatible? :line
  [first-card second-card]
  (area-bar-line-series-are-compatible? first-card second-card))

(defmethod series-are-compatible? :bar
  [first-card second-card]
  (area-bar-line-series-are-compatible? first-card second-card))

(defmethod series-are-compatible? :scalar
  [first-card second-card]
  (and (= :scalar (:display second-card))
       (= 1
          (count (:result_metadata first-card))
          (count (:result_metadata second-card)))))

(def ^:private supported-series-display-type (set (keys (methods series-are-compatible?))))

(defn- fetch-compatible-series*
  [card {:keys [query last-cursor page-size exclude-ids] :as _options}]
  (let [matching-cards  (t2/select Card
                                   :archived false
                                   :display [:in supported-series-display-type]
                                   :id [:not= (:id card)]
                                   (cond-> {:order-by [[:id :desc]]
                                            :where    [:and]}
                                     last-cursor
                                     (update :where conj [:< :id last-cursor])
                                     (seq exclude-ids)
                                     (update :where conj [:not [:in :id exclude-ids]])
                                     query
                                     (update :where conj [:like :%lower.name (str "%" (u/lower-case-en query) "%")])
                                     ;; add a little buffer to the page to account for cards that are not
                                     ;; compatible + do not have permissions to read
                                     ;; this is just a heuristic, but it should be good enough
                                     page-size
                                     (assoc :limit (+ 10 page-size))))
        compatible-cards (->> matching-cards
                              (filter mi/can-read?)
                              (filter #(or
                                         ;; columns name on native query are not match with the column name in viz-settings. why??
                                         ;; so we can't use series-are-compatible? to filter out incompatible native cards.
                                         ;; => we assume all native queries are compatible and FE will figure it out later
                                         (= (:query_type %) :native)
                                         (series-are-compatible? card %))))]
    (if page-size
      (take page-size compatible-cards)
      compatible-cards)))

(defn- fetch-compatible-series
  ([card options]
   (fetch-compatible-series card options []))
  ([card {:keys [page-size] :as options} current-cards]
   (let [cards     (fetch-compatible-series* card options)
         new-cards (concat current-cards cards)]
     ;; if the total card fetches is less than page-size and there are still more, continue fetching
     (if (and (some? page-size)
              (seq cards)
              (< (count cards) page-size))
       (fetch-compatible-series card
                                (merge options
                                       {:page-size   (- page-size (count cards))
                                        :last-cursor (:id (last cards))})
                                new-cards)
       new-cards))))

(api/defendpoint GET 
  "Fetches a list of comptatible series with the card with id `card_id`.
  - `last_cursor` with value is the id of the last card from the previous page to fetch the next page.
  - `query` to search card by name.
  - `exclude_ids` to filter out a list of card ids"
  [id last_cursor query exclude_ids]
  {id          int?
   last_cursor [:maybe ms/PositiveInt]
   query       [:maybe ms/NonBlankString]
   exclude_ids [:maybe [:fn
                        {:error/fn (fn [_ _] (deferred-tru "value must be a sequence of positive integers"))}
                        (fn [ids]
                          (every? pos-int? (api/parse-multi-values-param ids parse-long)))]]}
  (let [exclude_ids  (when exclude_ids (api/parse-multi-values-param exclude_ids parse-long))
        card         (-> (t2/select-one :model/Card :id id) api/check-404 api/read-check)
        card-display (:display card)]
   (when-not (supported-series-display-type card-display)
             (throw (ex-info (tru "Card with type {0} is not compatible to have series" (name card-display))
                             {:display         card-display
                              :allowed-display (map name supported-series-display-type)
                              :status-code     400})))
   (fetch-compatible-series
     card
     {:exclude-ids exclude_ids
      :query       query
      :last-cursor last_cursor
      :page-size   mw.offset-paging/*limit*})))

(api/defendpoint GET 
  "Get the timelines for card with ID. Looks up the collection the card is in and uses that."
  [id include start end]
  {id      ms/PositiveInt
   include [:maybe [:= "events"]]
   start   [:maybe ms/TemporalString]
   end     [:maybe ms/TemporalString]}
  (let [{:keys [collection_id] :as _card} (api/read-check Card id)]
    ;; subtlety here. timeline access is based on the collection at the moment so this check should be identical. If
    ;; we allow adding more timelines to a card in the future, we will need to filter on read-check and i don't think
    ;; the read-checks are particularly fast on multiple items
    (timeline/timelines-for-collection collection_id
                                       {:timeline/events? (= include "events")
                                        :events/start     (when start (u.date/parse start))
                                        :events/end       (when end (u.date/parse end))})))

(defn check-data-permissions-for-query
  [query]
  {:pre [(map? query)]}
  (when-not (query-perms/can-run-query? query)
    (let [required-perms (try
                           (query-perms/perms-set query :throw-exceptions? true)
                           (catch Throwable e
                             e))]
      (throw (ex-info (tru "You cannot save this Question because you do not have permissions to run its query.")
                      {:status-code    403
                       :query          query
                       :required-perms (if (instance? Throwable required-perms)
                                         :error
                                         required-perms)
                       :actual-perms   @api/*current-user-permissions-set*}
                      (when (instance? Throwable required-perms)
                        required-perms))))))

(api/defendpoint POST 
  "Create a new `Card`."
  [:as {{:keys [collection_id collection_position dataset dataset_query description display name
                parameters parameter_mappings result_metadata visualization_settings cache_ttl], :as body} :body}]
  {name                   ms/NonBlankString
   dataset                [:maybe :boolean]
   dataset_query          ms/Map
   parameters             [:maybe [:sequential ms/Parameter]]
   parameter_mappings     [:maybe [:sequential ms/ParameterMapping]]
   description            [:maybe ms/NonBlankString]
   display                ms/NonBlankString
   visualization_settings ms/Map
   collection_id          [:maybe ms/PositiveInt]
   collection_position    [:maybe ms/PositiveInt]
   result_metadata        [:maybe qr/ResultsMetadata]
   cache_ttl              [:maybe ms/PositiveInt]}
  ;; check that we have permissions to run the query that we're trying to save
  (check-data-permissions-for-query dataset_query)
  ;; check that we have permissions for the collection we're trying to save this card to, if applicable
  (collection/check-write-perms-for-collection collection_id)
  (-> (card/create-card! body @api/*current-user*)
      hydrate-card-details
      (assoc :last-edit-info (last-edit/edit-information-for-user @api/*current-user*))))

(api/defendpoint POST 
  "Copy a `Card`, with the new name 'Copy of _name_'"
  [id]
  {id [:maybe ms/PositiveInt]}
  (let [orig-card (api/read-check Card id)
        new-name  (str (trs "Copy of ") (:name orig-card))
        new-card  (assoc orig-card :name new-name)]
    (-> (card/create-card! new-card @api/*current-user*)
        hydrate-card-details
        (assoc :last-edit-info (last-edit/edit-information-for-user @api/*current-user*)))))

(defn- check-allowed-to-modify-query
  [card-before-updates card-updates]
  (let [card-updates (m/update-existing card-updates :dataset_query mbql.normalize/normalize)]
    (when (api/column-will-change? :dataset_query card-before-updates card-updates)
      (check-data-permissions-for-query (:dataset_query card-updates)))))

(defn- check-allowed-to-change-embedding
  [card-before-updates card-updates]
  (when (or (api/column-will-change? :enable_embedding card-before-updates card-updates)
            (api/column-will-change? :embedding_params card-before-updates card-updates))
    (validation/check-embedding-enabled)
    (api/check-superuser)))

(api/defendpoint PUT 
  "Update a `Card`."
  [id :as {{:keys [dataset_query description display name visualization_settings archived collection_id
                   collection_position enable_embedding embedding_params result_metadata parameters
                   cache_ttl dataset collection_preview]
            :as   card-updates} :body}]
  {id                     ms/PositiveInt
   name                   [:maybe ms/NonBlankString]
   parameters             [:maybe [:sequential ms/Parameter]]
   dataset_query          [:maybe ms/Map]
   dataset                [:maybe :boolean]
   display                [:maybe ms/NonBlankString]
   description            [:maybe :string]
   visualization_settings [:maybe ms/Map]
   archived               [:maybe :boolean]
   enable_embedding       [:maybe :boolean]
   embedding_params       [:maybe ms/EmbeddingParams]
   collection_id          [:maybe ms/PositiveInt]
   collection_position    [:maybe ms/PositiveInt]
   result_metadata        [:maybe qr/ResultsMetadata]
   cache_ttl              [:maybe ms/PositiveInt]
   collection_preview     [:maybe :boolean]}
  (let [card-before-update (t2/hydrate (api/write-check Card id)
                                       [:moderation_reviews :moderator_details])]
    ;; Do various permissions checks
    (doseq [f [collection/check-allowed-to-change-collection
               check-allowed-to-modify-query
               check-allowed-to-change-embedding]]
      (f card-before-update card-updates))
    ;; make sure we have the correct `result_metadata`
    (let [result-metadata-chan  (card/result-metadata-async {:original-query    (:dataset_query card-before-update)
                                                             :query             dataset_query
                                                             :metadata          result_metadata
                                                             :original-metadata (:result_metadata card-before-update)
                                                             :dataset?          (if (some? dataset)
                                                                                  dataset
                                                                                  (:dataset card-before-update))})
          card-updates          (merge card-updates
                                       (when dataset
                                         {:display :table}))
          metadata-timeout      (a/timeout card/metadata-sync-wait-ms)
          [fresh-metadata port] (a/alts!! [result-metadata-chan metadata-timeout])
          timed-out?            (= port metadata-timeout)
          card-updates          (cond-> card-updates
                                  (not timed-out?)
                                  (assoc :result_metadata fresh-metadata))]
      (u/prog1 (-> (card/update-card! {:card-before-update card-before-update
                                       :card-updates       card-updates
                                       :actor              @api/*current-user*})
                   hydrate-card-details
                   (assoc :last-edit-info (last-edit/edit-information-for-user @api/*current-user*)))
        (when timed-out?
          (log/info (trs "Metadata not available soon enough. Saving card {0} and asynchronously updating metadata" id))
          (card/schedule-metadata-saving result-metadata-chan <>))))))

(api/defendpoint DELETE 
  "Delete a Card. (DEPRECATED -- don't delete a Card anymore -- archive it instead.)"
  [id]
  {id ms/PositiveInt}
  (log/warn (tru "DELETE /api/card/:id is deprecated. Instead, change its `archived` value via PUT /api/card/:id."))
  (let [card (api/write-check Card id)]
    (t2/delete! Card :id id)
    (events/publish-event! :event/card-delete {:object card :user-id api/*current-user-id*}))
  api/generic-204-no-content)

(defn- update-collection-positions!
  [new-collection-id-or-nil cards]
  ;; Sorting by `:collection_position` to ensure lower position cards are appended first
  (let [sorted-cards        (sort-by :collection_position cards)
        max-position-result (t2/select-one [Card [:%max.collection_position :max_position]]
                              :collection_id new-collection-id-or-nil)
        ;; collection_position for the next card in the collection
        starting-position   (inc (get max-position-result :max_position 0))]
    ;; This is using `map` but more like a `doseq` with multiple seqs. Wrapping this in a `doall` as we don't want it
    ;; to be lazy and we're just going to discard the results
    (doall
     (map (fn [idx {:keys [collection_id collection_position] :as card}]
            ;; We are removing this card from `collection_id` so we need to reconcile any
            ;; `collection_position` entries left behind by this move
            (api/reconcile-position-for-collection! collection_id collection_position nil)
            ;; Now we can update the card with the new collection and a new calculated position
            ;; that appended to the end
            (t2/update! Card
                        (u/the-id card)
                        {:collection_position idx
                         :collection_id       new-collection-id-or-nil}))
          ;; These are reversed because of the classic issue when removing an item from array. If we remove an
          ;; item at index 1, everthing above index 1 will get decremented. By reversing our processing order we
          ;; can avoid changing the index of cards we haven't yet updated
          (reverse (range starting-position (+ (count sorted-cards) starting-position)))
          (reverse sorted-cards)))))

(defn- move-cards-to-collection! [new-collection-id-or-nil card-ids]
  ;; if moving to a collection, make sure we have write perms for it
  (when new-collection-id-or-nil
    (api/write-check Collection new-collection-id-or-nil))
  ;; for each affected card...
  (when (seq card-ids)
    (let [cards (t2/select [Card :id :collection_id :collection_position :dataset_query]
                  {:where [:and [:in :id (set card-ids)]
                                [:or [:not= :collection_id new-collection-id-or-nil]
                                  (when new-collection-id-or-nil
                                    [:= :collection_id nil])]]})] ; poisioned NULLs = ick
      ;; ...check that we have write permissions for it...
      (doseq [card cards]
        (api/write-check card))
      ;; ...and check that we have write permissions for the old collections if applicable
      (doseq [old-collection-id (set (filter identity (map :collection_id cards)))]
        (api/write-check Collection old-collection-id))
      ;; Ensure all of the card updates occur in a transaction. Read commited (the default) really isn't what we want
      ;; here. We are querying for the max card position for a given collection, then using that to base our position
      ;; changes if the cards are moving to a different collection. Without repeatable read here, it's possible we'll
      ;; get duplicates
      (t2/with-transaction [_conn]
        ;; If any of the cards have a `:collection_position`, we'll need to fixup the old collection now that the cards
        ;; are gone and update the position in the new collection
        (when-let [cards-with-position (seq (filter :collection_position cards))]
          (update-collection-positions! new-collection-id-or-nil cards-with-position))
        ;; ok, everything checks out. Set the new `collection_id` for all the Cards that haven't been updated already
        (when-let [cards-without-position (seq (for [card cards
                                                     :when (not (:collection_position card))]
                                                 (u/the-id card)))]
          (t2/update! (t2/table-name Card)
                      {:id [:in (set cards-without-position)]}
                      {:collection_id new-collection-id-or-nil}))))))

(api/defendpoint POST 
  "Bulk update endpoint for Card Collections. Move a set of `Cards` with `card_ids` into a `Collection` with
  `collection_id`, or remove them from any Collections by passing a `null` `collection_id`."
  [:as {{:keys [card_ids collection_id]} :body}]
  {card_ids      [:sequential ms/PositiveInt]
   collection_id [:maybe ms/PositiveInt]}
  (move-cards-to-collection! collection_id card_ids)
  {:status :ok})

(api/defendpoint POST 
  "Run the query associated with a Card."
  [card-id :as {{:keys [parameters ignore_cache dashboard_id collection_preview], :or {ignore_cache false dashboard_id nil}} :body}]
  {card-id            ms/PositiveInt
   ignore_cache       [:maybe :boolean]
   collection_preview [:maybe :boolean]
   dashboard_id       [:maybe ms/PositiveInt]}
  ;; TODO -- we should probably warn if you pass `dashboard_id`, and tell you to use the new
  ;;
  ;;    POST /api/dashboard/:dashboard-id/card/:card-id/query
  ;;
  ;; endpoint instead. Or error in that situtation? We're not even validating that you have access to this Dashboard.
  (qp.card/run-query-for-card-async
   card-id :api
   :parameters   parameters
   :ignore_cache ignore_cache
   :dashboard-id dashboard_id
   :context      (if collection_preview :collection :question)
   :middleware   {:process-viz-settings? false}))

(api/defendpoint POST 
  "Run the query associated with a Card, and return its results as a file in the specified format.
  `parameters` should be passed as query parameter encoded as a serialized JSON string (this is because this endpoint
  is normally used to power 'Download Results' buttons that use HTML `form` actions)."
  [card-id export-format :as {{:keys [parameters]} :params}]
  {card-id       ms/PositiveInt
   parameters    [:maybe ms/JSONString]
   export-format (into [:enum] api.dataset/export-formats)}
  (qp.card/run-query-for-card-async
   card-id export-format
   :parameters  (json/parse-string parameters keyword)
   :constraints nil
   :context     (api.dataset/export-format->context export-format)
   :middleware  {:process-viz-settings?  true
                 :skip-results-metadata? true
                 :ignore-cached-results? true
                 :format-rows?           false
                 :js-int-to-string?      false}))

(api/defendpoint POST 
  "Generate publicly-accessible links for this Card. Returns UUID to be used in public links. (If this Card has
  already been shared, it will return the existing public link rather than creating a new one.)  Public sharing must
  be enabled."
  [card-id]
  {card-id ms/PositiveInt}
  (validation/check-has-application-permission :setting)
  (validation/check-public-sharing-enabled)
  (api/check-not-archived (api/read-check Card card-id))
  (let [{existing-public-uuid :public_uuid} (t2/select-one [Card :public_uuid] :id card-id)]
    {:uuid (or existing-public-uuid
               (u/prog1 (str (random-uuid))
                 (t2/update! Card card-id
                             {:public_uuid       <>
                              :made_public_by_id api/*current-user-id*})))}))

(api/defendpoint DELETE 
  "Delete the publicly-accessible link to this Card."
  [card-id]
  {card-id ms/PositiveInt}
  (validation/check-has-application-permission :setting)
  (validation/check-public-sharing-enabled)
  (api/check-exists? Card :id card-id, :public_uuid [:not= nil])
  (t2/update! Card card-id
              {:public_uuid       nil
               :made_public_by_id nil})
  {:status 204, :body nil})

(api/defendpoint GET 
  "Fetch a list of Cards with public UUIDs. These cards are publicly-accessible *if* public sharing is enabled."
  []
  (validation/check-has-application-permission :setting)
  (validation/check-public-sharing-enabled)
  (t2/select [Card :name :id :public_uuid], :public_uuid [:not= nil], :archived false))

(api/defendpoint GET 
  "Fetch a list of Cards where `enable_embedding` is `true`. The cards can be embedded using the embedding endpoints
  and a signed JWT."
  []
  (validation/check-has-application-permission :setting)
  (validation/check-embedding-enabled)
  (t2/select [Card :name :id], :enable_embedding true, :archived false))

(api/defendpoint GET 
  "Return related entities."
  [id]
  {id ms/PositiveInt}
  (-> (t2/select-one Card :id id) api/read-check related/related))

(api/defendpoint POST 
  "Return related entities for an ad-hoc query."
  [:as {query :body}]
  (related/related (query/adhoc-query query)))

(api/defendpoint POST 
  "Run the query associated with a Card."
  [card-id :as {{:keys [parameters ignore_cache]
                 :or   {ignore_cache false}} :body}]
  {card-id      ms/PositiveInt
   ignore_cache [:maybe :boolean]}
  (qp.card/run-query-for-card-async card-id :api
                            :parameters parameters,
                            :qp-runner qp.pivot/run-pivot-query
                            :ignore_cache ignore_cache))

(api/defendpoint POST 
  "Mark the model (card) as persisted. Runs the query and saves it to the database backing the card and hot swaps this
  query in place of the model's query."
  [card-id]
  {card-id ms/PositiveInt}
  (premium-features/assert-has-feature :cache-granular-controls (tru "Granular cache controls"))
  (api/let-404 [{:keys [dataset database_id] :as card} (t2/select-one Card :id card-id)]
    (let [database (t2/select-one Database :id database_id)]
      (api/write-check database)
      (when-not (driver/database-supports? (:engine database)
                                           :persist-models database)
        (throw (ex-info (tru "Database does not support persisting")
                        {:status-code 400
                         :database    (:name database)})))
      (when-not (driver/database-supports? (:engine database)
                                           :persist-models-enabled database)
        (throw (ex-info (tru "Persisting models not enabled for database")
                        {:status-code 400
                         :database    (:name database)})))
      (when-not dataset
        (throw (ex-info (tru "Card is not a model") {:status-code 400})))
      (when-let [persisted-info (persisted-info/turn-on-model! api/*current-user-id* card)]
        (task.persist-refresh/schedule-refresh-for-individual! persisted-info))
      api/generic-204-no-content)))

(api/defendpoint POST 
  "Refresh the persisted model caching `card-id`."
  [card-id]
  {card-id ms/PositiveInt}
  (api/let-404 [card           (t2/select-one Card :id card-id)
                persisted-info (t2/select-one PersistedInfo :card_id card-id)]
    (when (not (:dataset card))
      (throw (ex-info (trs "Cannot refresh a non-model question") {:status-code 400})))
    (when (:archived card)
      (throw (ex-info (trs "Cannot refresh an archived model") {:status-code 400})))
    (api/write-check (t2/select-one Database :id (:database_id persisted-info)))
    (task.persist-refresh/schedule-refresh-for-individual! persisted-info)
    api/generic-204-no-content))

(api/defendpoint POST 
  "Unpersist this model. Deletes the persisted table backing the model and all queries after this will use the card's
  query rather than the saved version of the query."
  [card-id]
  {card-id ms/PositiveInt}
  (premium-features/assert-has-feature :cache-granular-controls (tru "Granular cache controls"))
  (api/let-404 [_card (t2/select-one Card :id card-id)]
    (api/let-404 [persisted-info (t2/select-one PersistedInfo :card_id card-id)]
      (api/write-check (t2/select-one Database :id (:database_id persisted-info)))
      (persisted-info/mark-for-pruning! {:id (:id persisted-info)} "off")
      api/generic-204-no-content)))

(defn mapping->field-values
  [card param query]
  (when-let [field-clause (params/param-target->field-clause (:target param) card)]
    (when-let [field-id (mbql.u/match-one field-clause [:field (id :guard integer?) _] id)]
      (api.field/search-values-from-field-id field-id query))))

(mu/defn param-values
  ([card param-key]
   (param-values card param-key nil))
  ([card      :- ms/Map
    param-key :- ms/NonBlankString
    query     :- [:maybe ms/NonBlankString]]
   (let [param (get (m/index-by :id (or (seq (:parameters card))
                                        ;; some older cards or cards in e2e just use the template tags on native queries
                                        (card/template-tag-parameters card)))
                    param-key)]
     (when-not param
       (throw (ex-info (tru "Card does not have a parameter with the ID {0}" (pr-str param-key))
                       {:status-code 400})))
     (custom-values/parameter->values param query (fn [] (mapping->field-values card param query))))))

(api/defendpoint GET 
  "Fetch possible values of the parameter whose ID is `:param-key`.
    ;; fetch values for Card 1 parameter 'abc' that are possible
    GET /api/card/1/params/abc/values"
  [card-id param-key]
  {card-id   ms/PositiveInt
   param-key ms/NonBlankString}
  (param-values (api/read-check Card card-id) param-key))

(api/defendpoint GET 
  "Fetch possible values of the parameter whose ID is `:param-key` that contain `:query`.
    ;; fetch values for Card 1 parameter 'abc' that contain 'Orange';
     GET /api/card/1/params/abc/search/Orange
  Currently limited to first 1000 results."
  [card-id param-key query]
  {card-id   ms/PositiveInt
   param-key ms/NonBlankString
   query     ms/NonBlankString}
  (param-values (api/read-check Card card-id) param-key query))

(defn- from-csv!
  [{:keys [collection-id filename file]}]
  (try
    (let [model (upload/create-csv-upload! {:collection-id collection-id
                                            :filename      filename
                                            :file          file
                                            :schema-name   (public-settings/uploads-schema-name)
                                            :table-prefix  (public-settings/uploads-table-prefix)
                                            :db-id         (or (public-settings/uploads-database-id)
                                                               (throw (ex-info (tru "The uploads database is not configured.")
                                                                               {:status-code 422})))})]
      {:status 200
       :body   (:id model)})
    (catch Throwable e
      {:status (or (-> e ex-data :status-code)
                   500)
       :body   {:message (or (ex-message e)
                             (tru "There was an error uploading the file"))}})
    (finally (io/delete-file file :silently))))

(api/defendpoint ^:multipart POST 
  "Create a table and model populated with the values from the attached CSV. Returns the model ID if successful."
  [:as {raw-params :params}]
  ;; parse-long returns nil with "root" as the collection ID, which is what we want anyway
  (from-csv! {:collection-id (parse-long (get raw-params "collection_id"))
              :filename      (get-in raw-params ["file" :filename])
              :file          (get-in raw-params ["file" :tempfile])}))

(api/define-routes)

(ns metabase.api.collection
  (:require
   [cheshire.core :as json]
   [clojure.string :as str]
   [compojure.core :refer [GET POST PUT]]
   [honey.sql.helpers :as sql.helpers]
   [malli.core :as mc]
   [malli.transform :as mtx]
   [medley.core :as m]
   [metabase.api.common :as api]
   [metabase.db :as mdb]
   [metabase.db.query :as mdb.query]
   [metabase.driver.common.parameters :as params]
   [metabase.driver.common.parameters.parse :as params.parse]
   [metabase.mbql.normalize :as mbql.normalize]
   [metabase.models.card :as card :refer [Card]]
   [metabase.models.collection :as collection :refer [Collection]]
   [metabase.models.collection.graph :as graph]
   [metabase.models.collection.root :as collection.root]
   [metabase.models.dashboard :refer [Dashboard]]
   [metabase.models.interface :as mi]
   [metabase.models.native-query-snippet :refer [NativeQuerySnippet]]
   [metabase.models.permissions :as perms]
   [metabase.models.pulse :as pulse :refer [Pulse]]
   [metabase.models.revision.last-edit :as last-edit]
   [metabase.models.timeline :as timeline :refer [Timeline]]
   [metabase.public-settings.premium-features
    :as premium-features
    :refer [defenterprise]]
   [metabase.server.middleware.offset-paging :as mw.offset-paging]
   [metabase.util :as u]
   [metabase.util.honey-sql-2 :as h2x]
   [metabase.util.i18n :refer [tru]]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(comment collection.root/keep-me)

(declare root-collection)

(defn- remove-other-users-personal-collections
  [user-id collections]
  (let [personal-ids (into #{} (comp (filter :personal_owner_id)
                                     (remove (comp #{user-id} :personal_owner_id))
                                     (map :id))
                           collections)
        prefixes     (into #{} (map (fn [id] (format "/%d/" id))) personal-ids)
        personal?    (fn [{^String location :location id :id}]
                       (or (personal-ids id)
                           (prefixes (re-find #"^/\d+/" location))))]
    (if (seq prefixes)
      (remove personal? collections)
      collections)))

(api/defendpoint GET 
  "Fetch a list of all Collections that the current user has read permissions for (`:can_write` is returned as an
  additional property of each Collection so you can tell which of these you have write permissions for.)
  By default, this returns non-archived Collections, but instead you can show archived ones by passing
  `?archived=true`.
  By default, admin users will see all collections. To hide other user's collections pass in
  `?exclude-other-user-collections=true`."
  [archived exclude-other-user-collections namespace]
  {archived                       [:maybe ms/BooleanValue]
   exclude-other-user-collections [:maybe ms/BooleanValue]
   namespace                      [:maybe ms/NonBlankString]}
  (as-> (t2/select Collection
                   {:where    [:and
                               [:= :archived archived]
                               (perms/audit-namespace-clause :namespace namespace)
                               (collection/visible-collection-ids->honeysql-filter-clause
                                :id
                                (collection/permissions-set->visible-collection-ids @api/*current-user-permissions-set*))]
                              ;; Order NULL collection types first so that audit collections are last
                    :order-by [[[[:case [:= :type nil] 0 :else 1]] :asc]
                               [:%lower.name :asc]]}) collections
    ;; Remove other users' personal collections
    (if exclude-other-user-collections
      (remove-other-users-personal-collections api/*current-user-id* collections)
      collections)
    ;; include Root Collection at beginning or results if archived isn't `true`
    (if archived
      collections
      (let [root (root-collection namespace)]
        (cond->> collections
          (mi/can-read? root)
          (cons root))))
    (t2/hydrate collections :can_write :is_personal)
    ;; remove the :metabase.models.collection.root/is-root? tag since FE doesn't need it
    ;; and for personal collections we translate the name to user's locale
    (for [collection collections]
      (-> collection
          (dissoc ::collection.root/is-root?)
          collection/personal-collection-with-ui-details))))

(defn- shallow-tree-from-collection-id
  [colls]
  (->> colls
       (map collection/personal-collection-with-ui-details)
       (collection/collections->tree nil)
       (map (fn [coll] (update coll :children #(boolean (seq %)))))))

(defn- location-from-collection-id-clause
  [collection-id]
  (if collection-id
    [:and
     [:like :location (str "%/" collection-id "/%")]
     [:not [:like :location (str "%/" collection-id "/%/%/%")]]]
    [:not [:like :location "/%/%/"]]))

(defn- select-collections
  [exclude-archived exclude-other-user-collections namespace shallow collection-id]
  (cond->>
   (t2/select Collection
              {:where [:and
                       (when exclude-archived
                         [:= :archived false])
                       (when shallow
                         (location-from-collection-id-clause collection-id))
                       (perms/audit-namespace-clause :namespace namespace)
                       (collection/visible-collection-ids->honeysql-filter-clause
                        :id
                        (collection/permissions-set->visible-collection-ids @api/*current-user-permissions-set*))]})
    exclude-other-user-collections (remove-other-users-personal-collections api/*current-user-id*)))

(api/defendpoint GET 
  "Similar to `GET /`, but returns Collections in a tree structure, e.g.
  ```
  [{:name     \"A\"
  :below    #{:card :dataset}
  :children [{:name \"B\"}
             {:name     \"C\"
              :here     #{:dataset :card}
              :below    #{:dataset :card}
              :children [{:name     \"D\"
                          :here     #{:dataset}
                          :children [{:name \"E\"}]}
                         {:name     \"F\"
                          :here     #{:card}
                          :children [{:name \"G\"}]}]}]}
  {:name \"H\"}]
  ```
  The here and below keys indicate the types of items at this particular level of the tree (here) and in its
  subtree (below)."
  [exclude-archived exclude-other-user-collections namespace shallow collection-id]
  {exclude-archived               [:maybe :boolean]
   exclude-other-user-collections [:maybe :boolean]
   namespace                      [:maybe ms/NonBlankString]
   shallow                        [:maybe :boolean]
   collection-id                  [:maybe ms/PositiveInt]}
  (let [collections (select-collections exclude-archived exclude-other-user-collections namespace shallow collection-id)]
    (if shallow
      (shallow-tree-from-collection-id collections)
      (let [collection-type-ids (reduce (fn [acc {:keys [collection_id dataset] :as _x}]
                                          (update acc (if dataset :dataset :card) conj collection_id))
                                        {:dataset #{}
                                         :card    #{}}
                                        (mdb.query/reducible-query {:select-distinct [:collection_id :dataset]
                                                                    :from            [:report_card]
                                                                    :where           [:= :archived false]}))
            collections-with-details (map collection/personal-collection-with-ui-details collections)]
        (collection/collections->tree collection-type-ids collections-with-details)))))

(def ^:private valid-model-param-values
  #{"card" "dataset" "collection" "dashboard" "pulse" "snippet" "no_models" "timeline"})

(def ^:private ModelString
  (into [:enum] valid-model-param-values))

(def ^:private Models
  [:or
   [:sequential ModelString]
   ModelString])

(def ^:private valid-pinned-state-values
  #{"all" "is_pinned" "is_not_pinned"})

(def ^:private valid-sort-columns #{"name" "last_edited_at" "last_edited_by" "model"})
(def ^:private valid-sort-directions #{"asc" "desc"})
(defn- normalize-sort-choice [w] (when w (keyword (str/replace w #"_" "-"))))

(def ^:private CollectionChildrenOptions
  [:map
   [:archived?                     :boolean]
   [:pinned-state {:optional true} [:maybe (into [:enum] (map keyword) valid-pinned-state-values)]]
   ;; when specified, only return results of this type.
   [:models       {:optional true} [:maybe [:set (into [:enum] (map keyword) valid-model-param-values)]]]
   [:sort-info    {:optional true} [:maybe [:tuple
                                            (into [:enum {:error/message "sort-columns"}]
                                                  (map normalize-sort-choice)
                                                  valid-sort-columns)
                                            (into [:enum {:error/message "sort-direction"}]
                                                  (map normalize-sort-choice)
                                                  valid-sort-directions)]]]])

(defmulti ^:private collection-children-query
  {:arglists '([model collection options])}
  (fn [model _ _] (keyword model)))

(def ^:private always-true-hsql-expr
  [:= [:inline 1] [:inline 1]])

(def ^:private always-false-hsql-expr
  [:= [:inline 1] [:inline 2]])

(defn- pinned-state->clause
  ([pinned-state]
   (pinned-state->clause pinned-state :collection_position))
  ([pinned-state col]
   (case pinned-state
     :all           always-true-hsql-expr
     :is_pinned     [:<> col nil]
     :is_not_pinned [:= col nil]
     always-true-hsql-expr)))

(defn- poison-when-pinned-clause
  [pinned-state]
  (if (= pinned-state :is_pinned)
    always-false-hsql-expr
    always-true-hsql-expr))

(defmulti ^:private post-process-collection-children
  {:arglists '([model rows])}
  (fn [model _]
    (keyword model)))

(defmethod ^:private post-process-collection-children :default
  [_ rows]
  rows)

(defmethod collection-children-query :pulse
  [_ collection {:keys [archived? pinned-state]}]
  (-> {:select-distinct [:p.id
                         :p.name
                         :p.entity_id
                         :p.collection_position
                         [(h2x/literal "pulse") :model]]
       :from            [[:pulse :p]]
       :left-join       [[:pulse_card :pc] [:= :p.id :pc.pulse_id]]
       :where           [:and
                         [:= :p.collection_id      (:id collection)]
                         [:= :p.archived           (boolean archived?)]
                         ;; exclude alerts
                         [:= :p.alert_condition    nil]
                         ;; exclude dashboard subscriptions
                         [:= :p.dashboard_id nil]]}
      (sql.helpers/where (pinned-state->clause pinned-state :p.collection_position))))

(defmethod post-process-collection-children :pulse
  [_ rows]
  (for [row rows]
    (dissoc row
            :description :display :authority_level :moderated_status :icon :personal_owner_id
            :collection_preview :dataset_query)))

(defenterprise snippets-collection-children-query
  metabase-enterprise.snippet-collections.api.native-query-snippet
  [_ {:keys [archived?]}]
  {:select [:id :name :entity_id [(h2x/literal "snippet") :model]]
   :from   [[:native_query_snippet :nqs]]
   :where  [:= :archived (boolean archived?)]})

(defmethod collection-children-query :snippet
  [_ collection options]
  (snippets-collection-children-query collection options))

(defmethod collection-children-query :timeline
  [_ collection {:keys [archived? pinned-state]}]
  {:select [:id :name [(h2x/literal "timeline") :model] :description :entity_id :icon]
   :from   [[:timeline :timeline]]
   :where  [:and
            (poison-when-pinned-clause pinned-state)
            [:= :collection_id (:id collection)]
            [:= :archived (boolean archived?)]]})

(defmethod post-process-collection-children :timeline
  [_ rows]
  (for [row rows]
    (dissoc row
            :description :display :collection_position :authority_level :moderated_status
            :collection_preview :dataset_query)))

(defmethod post-process-collection-children :snippet
  [_ rows]
  (for [row rows]
    (dissoc row
            :description :collection_position :display :authority_level
            :moderated_status :icon :personal_owner_id :collection_preview
            :dataset_query)))

(defn- card-query [dataset? collection {:keys [archived? pinned-state]}]
  (-> {:select    (cond->
                    [:c.id :c.name :c.description :c.entity_id :c.collection_position :c.display :c.collection_preview
                     :c.dataset_query
                     [(h2x/literal (if dataset? "dataset" "card")) :model]
                     [:u.id :last_edit_user]
                     [:u.email :last_edit_email]
                     [:u.first_name :last_edit_first_name]
                     [:u.last_name :last_edit_last_name]
                     [:r.timestamp :last_edit_timestamp]
                     [{:select   [:status]
                       :from     [:moderation_review]
                       :where    [:and
                                  [:= :moderated_item_type "card"]
                                  [:= :moderated_item_id :c.id]
                                  [:= :most_recent true]]
                       ;; limit 1 to ensure that there is only one result but this invariant should hold true, just
                       ;; protecting against potential bugs
                       :order-by [[:id :desc]]
                       :limit    1}
                      :moderated_status]]
                    dataset?
                    (conj :c.database_id))
       :from      [[:report_card :c]]
       :left-join [[:revision :r] [:and
                                   [:= :r.model_id :c.id]
                                   [:= :r.most_recent true]
                                   [:= :r.model (h2x/literal "Card")]]
                   [:core_user :u] [:= :u.id :r.user_id]]
       :where     [:and
                   [:= :collection_id (:id collection)]
                   [:= :archived (boolean archived?)]
                   [:= :dataset dataset?]]}
      (sql.helpers/where (pinned-state->clause pinned-state))))

(defmethod collection-children-query :dataset
  [_ collection options]
  (card-query true collection options))

(defmethod post-process-collection-children :dataset
  [_ rows]
  (post-process-collection-children :card rows))

(defmethod collection-children-query :card
  [_ collection options]
  (card-query false collection options))

(defn- fully-parametrized-text?
  [text template-tags]
  (try
    (let [obligatory-params (into #{}
                                  (comp (filter params/Param?)
                                        (map :k))
                                  (params.parse/parse text))]
      (and (every? #(or (#{:dimension :snippet} (:type %))
                        (:default %))
                   (map template-tags obligatory-params))
           (every? #(or (not (:required %))
                        (:default %))
                   (vals template-tags))))
    (catch clojure.lang.ExceptionInfo _
      ;; An exception might be thrown during parameter parsing if the syntax is invalid. In this case we return
      ;; true so that we still can try to generate a preview for the query and display an error.
      false)))

(defn- fully-parametrized-query? [row]
  (let [native-query (-> row :dataset_query json/parse-string mbql.normalize/normalize :native)]
    (if-let [template-tags (:template-tags native-query)]
      (fully-parametrized-text? (:query native-query) template-tags)
      true)))

(defn- post-process-card-row [row]
  (-> row
      (dissoc :authority_level :icon :personal_owner_id :dataset_query)
      (update :collection_preview api/bit->boolean)
      (assoc :fully_parametrized (fully-parametrized-query? row))))

(defmethod post-process-collection-children :card
  [_ rows]
  (map post-process-card-row rows))

(defn- dashboard-query [collection {:keys [archived? pinned-state]}]
  (-> {:select    [:d.id :d.name :d.description :d.entity_id :d.collection_position
                   [(h2x/literal "dashboard") :model]
                   [:u.id :last_edit_user]
                   [:u.email :last_edit_email]
                   [:u.first_name :last_edit_first_name]
                   [:u.last_name :last_edit_last_name]
                   [:r.timestamp :last_edit_timestamp]]
       :from      [[:report_dashboard :d]]
       :left-join [[:revision :r] [:and
                                   [:= :r.model_id :d.id]
                                   [:= :r.most_recent true]
                                   [:= :r.model (h2x/literal "Dashboard")]]
                   [:core_user :u] [:= :u.id :r.user_id]]
       :where     [:and
                   [:= :collection_id (:id collection)]
                   [:= :archived (boolean archived?)]]}
      (sql.helpers/where (pinned-state->clause pinned-state))))

(defmethod collection-children-query :dashboard
  [_ collection options]
  (dashboard-query collection options))

(defmethod post-process-collection-children :dashboard
  [_ rows]
  (map #(dissoc %
                :display :authority_level :moderated_status :icon :personal_owner_id :collection_preview
                :dataset_query)
       rows))

(defenterprise snippets-collection-filter-clause
  metabase-enterprise.snippet-collections.api.native-query-snippet
  []
  [:or
   [:= :namespace nil]
   [:not= :namespace (u/qualified-name "snippets")]])

(defn- collection-query
  [collection {:keys [archived? collection-namespace pinned-state]}]
  (-> (assoc (collection/effective-children-query
              collection
              [:= :archived archived?]
              (perms/audit-namespace-clause :namespace (u/qualified-name collection-namespace))
              (snippets-collection-filter-clause))
             ;; We get from the effective-children-query a normal set of columns selected:
             ;; want to make it fit the others to make UNION ALL work
             :select [:id
                      :name
                      :description
                      :entity_id
                      :personal_owner_id
                      [(h2x/literal "collection") :model]
                      :authority_level])
      ;; the nil indicates that collections are never pinned.
      (sql.helpers/where (pinned-state->clause pinned-state nil))))

(defmethod collection-children-query :collection
  [_ collection options]
  (collection-query collection options))

(defmethod post-process-collection-children :collection
  [_ rows]
  (letfn [(update-personal-collection [{:keys [personal_owner_id] :as row}]
            (if personal_owner_id
              ;; when fetching root collection, we might have personal collection
              (assoc row :name (collection/user->personal-collection-name (:personal_owner_id row) :user))
              (dissoc row :personal_owner_id)))]
    (for [row rows]
      ;; Go through this rigamarole instead of hydration because we
      ;; don't get models back from ulterior over-query
      ;; Previous examination with logging to DB says that there's no N+1 query for this.
      ;; However, this was only tested on H2 and Postgres
      (-> row
          (assoc :can_write (mi/can-write? Collection (:id row)))
          (dissoc :collection_position :display :moderated_status :icon
                  :collection_preview :dataset_query)
          update-personal-collection))))

(mu/defn ^:private coalesce-edit-info :- last-edit/MaybeAnnotated
  "Hoist all of the last edit information into a map under the key :last-edit-info. Considers this information present
  if `:last_edit_user` is not nil."
  [row]
  (letfn [(select-as [original k->k']
            (reduce (fn [m [k k']] (assoc m k' (get original k)))
                    {}
                    k->k'))]
    (let [mapping {:last_edit_user       :id
                   :last_edit_last_name  :last_name
                   :last_edit_first_name :first_name
                   :last_edit_email      :email
                   :last_edit_timestamp  :timestamp}]
      (cond-> (apply dissoc row :model_ranking (keys mapping))
        ;; don't use contains as they all have the key, we care about a value present
        (:last_edit_user row) (assoc :last-edit-info (select-as row mapping))))))

(defn- post-process-rows
  [rows]
  (->> (map-indexed (fn [i row] (vary-meta row assoc ::index i)) rows) ;; keep db sort order
       (group-by :model)
       (into []
             (comp (map (fn [[model rows]]
                          (post-process-collection-children (keyword model) rows)))
                   cat
                   (map coalesce-edit-info)))
       (sort-by (comp ::index meta))))

(defn- model-name->toucan-model [model-name]
  (case (keyword model-name)
    :collection Collection
    :card       Card
    :dataset    Card
    :dashboard  Dashboard
    :pulse      Pulse
    :snippet    NativeQuerySnippet
    :timeline   Timeline))

(defn- select-name
  [x]
  (if (vector? x)
    (recur (second x))
    (-> x name (str/split #"\.") peek keyword)))

(def ^:private all-select-columns
  [:id :name :description :entity_id :display [:collection_preview :boolean] :dataset_query
   :model :collection_position :authority_level [:personal_owner_id :integer]
   :last_edit_email :last_edit_first_name :last_edit_last_name :moderated_status :icon
   [:last_edit_user :integer] [:last_edit_timestamp :timestamp] [:database_id :integer]])

(defn- add-missing-columns
  [select-columns necessary-columns]
  (let [columns (m/index-by select-name select-columns)]
    (map (fn [col]
           (let [[col-name typpe] (u/one-or-many col)]
             (get columns col-name (if (and typpe (= (mdb/db-type) :postgres))
                                     [(h2x/cast typpe nil) col-name]
                                     [nil col-name]))))
         necessary-columns)))

(defn- add-model-ranking
  [select-clause model]
  (let [rankings {:dashboard  1
                  :pulse      2
                  :dataset    3
                  :card       4
                  :snippet    5
                  :collection 6
                  :timeline   7}]
    (conj select-clause [[:inline (get rankings model 100)]
                         :model_ranking])))

(comment
  ;; generate the set of columns across all child queries. Remember to add type info if not a text column
  (into []
        (comp cat (map select-name) (distinct))
        (for [model [:card :dashboard :snippet :pulse :collection :timeline]]
          (:select (collection-children-query model {:id 1 :location "/"} nil)))))

(defn children-sort-clause
  [sort-info db-type]
  (case sort-info
    nil                     [[:%lower.name :asc]]
    [:name :asc]            [[:%lower.name :asc]]
    [:name :desc]           [[:%lower.name :desc]]
    [:last-edited-at :asc]  [(if (= db-type :mysql)
                               [:%isnull.last_edit_timestamp]
                               [:last_edit_timestamp :nulls-last])
                             [:last_edit_timestamp :asc]
                             [:%lower.name :asc]]
    [:last-edited-at :desc] (remove nil?
                                    [(case db-type
                                       :mysql    [:%isnull.last_edit_timestamp]
                                       :postgres [:last_edit_timestamp :desc-nulls-last]
                                       :h2       nil)
                                     [:last_edit_timestamp :desc]
                                     [:%lower.name :asc]])
    [:last-edited-by :asc]  [(if (= db-type :mysql)
                               [:%isnull.last_edit_last_name]
                               [:last_edit_last_name :nulls-last])
                             [:last_edit_last_name :asc]
                             (if (= db-type :mysql)
                               [:%isnull.last_edit_first_name]
                               [:last_edit_first_name :nulls-last])
                             [:last_edit_first_name :asc]
                             [:%lower.name :asc]]
    [:last-edited-by :desc] (remove nil?
                                    [(case db-type
                                       :mysql    [:%isnull.last_edit_last_name]
                                       :postgres [:last_edit_last_name :desc-nulls-last]
                                       :h2       nil)
                                     [:last_edit_last_name :desc]
                                     (case db-type
                                       :mysql    [:%isnull.last_edit_first_name]
                                       :postgres [:last_edit_last_name :desc-nulls-last]
                                       :h2       nil)
                                     [:last_edit_first_name :desc]
                                     [:%lower.name :asc]])
    [:model :asc]           [[:model_ranking :asc]  [:%lower.name :asc]]
    [:model :desc]          [[:model_ranking :desc] [:%lower.name :asc]]))

(defn- collection-children*
  [collection models {:keys [sort-info] :as options}]
  (let [sql-order   (children-sort-clause sort-info (mdb/db-type))
        models      (sort (map keyword models))
        queries     (for [model models
                          :let  [query              (collection-children-query model collection options)
                                 select-clause-type (some
                                                     (fn [k]
                                                       (when (get query k)
                                                         k))
                                                     [:select :select-distinct])]]
                      (-> query
                          (update select-clause-type add-missing-columns all-select-columns)
                          (update select-clause-type add-model-ranking model)))
        total-query {:select [[:%count.* :count]]
                     :from   [[{:union-all queries} :dummy_alias]]}
        rows-query  {:select   [:*]
                     :from     [[{:union-all queries} :dummy_alias]]
                     :order-by sql-order}
        ;; We didn't implement collection pagination for snippets namespace for root/items
        ;; Rip out the limit for now and put it back in when we want it
        limit-query (if (or
                         (nil? mw.offset-paging/*limit*)
                         (nil? mw.offset-paging/*offset*)
                         (= (:collection-namespace options) "snippets"))
                      rows-query
                      (assoc rows-query
                             :limit  mw.offset-paging/*limit*
                             :offset mw.offset-paging/*offset*))
        res         {:total  (->> (mdb.query/query total-query) first :count)
                     :data   (->> (mdb.query/query limit-query) post-process-rows)
                     :models models}
        limit-res   (assoc res
                           :limit  mw.offset-paging/*limit*
                           :offset mw.offset-paging/*offset*)]
    (if (= (:collection-namespace options) "snippets")
      res
      limit-res)))

(mu/defn ^:private collection-children
  [{collection-namespace :namespace, :as collection} :- collection/CollectionWithLocationAndIDOrRoot
   {:keys [models], :as options}                     :- CollectionChildrenOptions]
  (let [valid-models (for [model-kw [:collection :dataset :card :dashboard :pulse :snippet :timeline]
                           ;; only fetch models that are specified by the `model` param; or everything if it's empty
                           :when    (or (empty? models) (contains? models model-kw))
                           :let     [toucan-model       (model-name->toucan-model model-kw)
                                     allowed-namespaces (collection/allowed-namespaces toucan-model)]
                           :when    (or (= model-kw :collection)
                                        (contains? allowed-namespaces (keyword collection-namespace)))]
                       model-kw)]
    (if (seq valid-models)
      (collection-children* collection valid-models (assoc options :collection-namespace collection-namespace))
      {:total  0
       :data   []
       :limit  mw.offset-paging/*limit*
       :offset mw.offset-paging/*offset*
       :models valid-models})))

(mu/defn ^:private collection-detail
  [collection :- collection/CollectionWithLocationAndIDOrRoot]
  (-> collection
      collection/personal-collection-with-ui-details
      (t2/hydrate :parent_id :effective_location [:effective_ancestors :can_write] :can_write :is_personal)))

(api/defendpoint GET 
  "Fetch a specific Collection with standard details added"
  [id]
  {id ms/PositiveInt}
  (collection-detail (api/read-check Collection id)))

(api/defendpoint GET 
  "Fetch the root Collection's timelines."
  [include archived]
  {include  [:maybe [:= "events"]]
   archived [:maybe :boolean]}
  (timeline/timelines-for-collection nil {:timeline/events?   (= include "events")
                                          :timeline/archived? archived}))

(api/defendpoint GET 
  "Fetch a specific Collection's timelines."
  [id include archived]
  {id       ms/PositiveInt
   include  [:maybe [:= "events"]]
   archived [:maybe :boolean]}
  (timeline/timelines-for-collection id {:timeline/events?   (= include "events")
                                         :timeline/archived? archived}))

(api/defendpoint GET 
  "Fetch a specific Collection's items with the following options:
  *  `models` - only include objects of a specific set of `models`. If unspecified, returns objects of all models
  *  `archived` - when `true`, return archived objects *instead* of unarchived ones. Defaults to `false`.
  *  `pinned_state` - when `is_pinned`, return pinned objects only.
                   when `is_not_pinned`, return non pinned objects only.
                   when `all`, return everything. By default returns everything"
  [id models archived pinned_state sort_column sort_direction]
  {id             ms/PositiveInt
   models         [:maybe Models]
   archived       [:maybe ms/BooleanString]
   pinned_state   [:maybe (into [:enum] valid-pinned-state-values)]
   sort_column    [:maybe (into [:enum] valid-sort-columns)]
   sort_direction [:maybe (into [:enum] valid-sort-directions)]}
  (let [model-kwds (set (map keyword (u/one-or-many models)))]
    (collection-children (api/read-check Collection id)
                         {:models       model-kwds
                          :archived?    (Boolean/parseBoolean archived)
                          :pinned-state (keyword pinned_state)
                          :sort-info    [(or (some-> sort_column normalize-sort-choice) :name)
                                         (or (some-> sort_direction normalize-sort-choice) :asc)]})))

(defn- root-collection [collection-namespace]
  (collection-detail (collection/root-collection-with-ui-details collection-namespace)))

(api/defendpoint GET 
  "Return the 'Root' Collection object with standard details added"
  [namespace]
  {namespace [:maybe ms/NonBlankString]}
  (-> (root-collection namespace)
      (api/read-check)
      (dissoc ::collection.root/is-root?)))

(defn- visible-model-kwds
  [root-collection model-set]
  (if (mi/can-read? root-collection)
    model-set
    (if (or (empty? model-set) (contains? model-set :collection))
      #{:collection}
      #{:no_models})))

(api/defendpoint GET 
  "Fetch objects that the current user should see at their root level. As mentioned elsewhere, the 'Root' Collection
  doesn't actually exist as a row in the application DB: it's simply a virtual Collection where things with no
  `collection_id` exist. It does, however, have its own set of Permissions.
  This endpoint will actually show objects with no `collection_id` for Users that have Root Collection
  permissions, but for people without Root Collection perms, we'll just show the objects that have an effective
  location of `/`.
  This endpoint is intended to power a 'Root Folder View' for the Current User, so regardless you'll see all the
  top-level objects you're allowed to access.
  By default, this will show the 'normal' Collections namespace; to view a different Collections namespace, such as
  `snippets`, you can pass the `?namespace=` parameter."
  [models archived namespace pinned_state sort_column sort_direction]
  {models         [:maybe Models]
   archived       [:maybe ms/BooleanString]
   namespace      [:maybe ms/NonBlankString]
   pinned_state   [:maybe (into [:enum] valid-pinned-state-values)]
   sort_column    [:maybe (into [:enum] valid-sort-columns)]
   sort_direction [:maybe (into [:enum] valid-sort-directions)]}
  ;; Return collection contents, including Collections that have an effective location of being in the Root
  ;; Collection for the Current User.
  (let [root-collection (assoc collection/root-collection :namespace namespace)
        model-set       (set (map keyword (u/one-or-many models)))
        model-kwds      (visible-model-kwds root-collection model-set)]
    (collection-children
     root-collection
     {:models       model-kwds
      :archived?    (Boolean/parseBoolean archived)
      :pinned-state (keyword pinned_state)
      :sort-info    [(or (some-> sort_column normalize-sort-choice) :name)
                     (or (some-> sort_direction normalize-sort-choice) :asc)]})))

(defn- write-check-collection-or-root-collection
  [collection-id collection-namespace]
  (api/write-check (if collection-id
                     (t2/select-one Collection :id collection-id)
                     (cond-> collection/root-collection
                       collection-namespace (assoc :namespace collection-namespace)))))

(defn create-collection!
  [{:keys [name description parent_id namespace authority_level]}]
  ;; To create a new collection, you need write perms for the location you are going to be putting it in...
  (write-check-collection-or-root-collection parent_id namespace)
  (when (some? authority_level)
    ;; make sure only admin and an EE token is present to be able to create an Official token
    (premium-features/assert-has-feature :official-collections (tru "Official Collections"))
    (api/check-superuser))
  ;; Now create the new Collection :)
  (first
    (t2/insert-returning-instances!
      Collection
      (merge
        {:name        name
         :description description
         :authority_level authority_level
         :namespace   namespace}
        (when parent_id
          {:location (collection/children-location (t2/select-one [Collection :location :id] :id parent_id))})))))

(api/defendpoint POST 
  "Create a new Collection."
  [:as {{:keys [name description parent_id namespace authority_level] :as body} :body}]
  {name            ms/NonBlankString
   description     [:maybe ms/NonBlankString]
   parent_id       [:maybe ms/PositiveInt]
   namespace       [:maybe ms/NonBlankString]
   authority_level [:maybe collection/AuthorityLevel]}
  (create-collection! body))

(defn- move-collection-if-needed!
  [collection-before-update collection-updates]
  ;; is a [new] parent_id update specified in the PUT request?
  (when (contains? collection-updates :parent_id)
    (let [orig-location (:location collection-before-update)
          new-parent-id (:parent_id collection-updates)
          new-parent    (if new-parent-id
                          (t2/select-one [Collection :location :id] :id new-parent-id)
                          collection/root-collection)
          new-location  (collection/children-location new-parent)]
      ;; check and make sure we're actually supposed to be moving something
      (when (not= orig-location new-location)
        ;; ok, make sure we have perms to do this operation
        (api/check-403
         (perms/set-has-full-permissions-for-set? @api/*current-user-permissions-set*
           (collection/perms-for-moving collection-before-update new-parent)))
        ;; ok, we're good to move!
        (collection/move-collection! collection-before-update new-location)))))

(defn- check-allowed-to-archive-or-unarchive
  [collection-before-update collection-updates]
  (when (api/column-will-change? :archived collection-before-update collection-updates)
    ;; Check that we have approprate perms
    (api/check-403
     (perms/set-has-full-permissions-for-set? @api/*current-user-permissions-set*
       (collection/perms-for-archiving collection-before-update)))))

(defn- maybe-send-archived-notifications!
  [& {:keys [collection-before-update collection-updates actor]}]
  (when (api/column-will-change? :archived collection-before-update collection-updates)
    (when-let [alerts (seq (pulse/retrieve-alerts-for-cards
                            {:card-ids (t2/select-pks-set Card :collection_id (u/the-id collection-before-update))}))]
      (card/delete-alert-and-notify-archived! {:alerts alerts :actor actor}))))

(api/defendpoint PUT 
  "Modify an existing Collection, including archiving or unarchiving it, or moving it."
  [id, :as {{:keys [name description archived parent_id authority_level], :as collection-updates} :body}]
  {id              ms/PositiveInt
   name            [:maybe ms/NonBlankString]
   description     [:maybe ms/NonBlankString]
   archived        [:maybe ms/BooleanValue]
   parent_id       [:maybe ms/PositiveInt]
   authority_level [:maybe collection/AuthorityLevel]}
  ;; do we have perms to edit this Collection?
  (let [collection-before-update (api/write-check Collection id)]
    ;; if we're trying to *archive* the Collection, make sure we're allowed to do that
    (check-allowed-to-archive-or-unarchive collection-before-update collection-updates)
    ;; if authority_level is changing, make sure we're allowed to do that
    (when (and (contains? collection-updates :authority_level)
               (not= (keyword authority_level) (:authority_level collection-before-update)))
      (premium-features/assert-has-feature :official-collections (tru "Official Collections"))
      (api/check-403 (and api/*is-superuser?*
                          ;; pre-update of model checks if the collection is a personal collection and rejects changes
                          ;; to authority_level, but it doesn't check if it is a sub-collection of a personal one so we add that
                          ;; here
                          (not (collection/is-personal-collection-or-descendant-of-one? collection-before-update)))))
    ;; ok, go ahead and update it! Only update keys that were specified in the `body`. But not `parent_id` since
    ;; that's not actually a property of Collection, and since we handle moving a Collection separately below.
    (let [updates (u/select-keys-when collection-updates :present [:name :description :archived :authority_level])]
      (when (seq updates)
        (t2/update! Collection id updates)))
    ;; if we're trying to *move* the Collection (instead or as well) go ahead and do that
    (move-collection-if-needed! collection-before-update collection-updates)
    ;; if we *did* end up archiving this Collection, we most post a few notifications
    (maybe-send-archived-notifications! {:collection-before-update collection-before-update
                                         :collection-updates       collection-updates
                                         :actor                    @api/*current-user*}))
  ;; finally, return the updated object
  (collection-detail (t2/select-one Collection :id id)))

(api/defendpoint GET 
  "Fetch a graph of all Collection Permissions."
  [namespace]
  {namespace [:maybe ms/NonBlankString]}
  (api/check-superuser)
  (graph/graph namespace))

(def CollectionID 
  [pos-int? {:title "Collection ID"}])

(def GroupID 
  [pos-int? {:title "Group ID"}])

(def CollectionPermissions
  [:and keyword? [:enum :write :read :none]])

(def GroupPermissionsGraph
  [:map-of
   [:or
    ;; We need the [:and keyword ...] piece to make decoding "root" work. There's a merged fix for this, but it hasn't
    ;; been released as of malli 0.9.2. When the malli version gets bumped, we should remove this.
    [:and keyword? [:= :root]]
    CollectionID]
   CollectionPermissions])

(def PermissionsGraph
  [:map
   [:revision int?]
   [:groups [:map-of GroupID GroupPermissionsGraph]]])

(def ^:private graph-decoder
  (mc/decoder PermissionsGraph (mtx/string-transformer)))

(defn- decode-graph [permission-graph]
  ;; TODO: should use a coercer for this?
  (graph-decoder permission-graph))

(api/defendpoint PUT 
  "Do a batch update of Collections Permissions by passing in a modified graph.
  Will overwrite parts of the graph that are present in the request, and leave the rest unchanged."
  [:as {{:keys [namespace], :as body} :body}]
  {body      :map
   namespace [:maybe ms/NonBlankString]}
  (api/check-superuser)
  (->> (dissoc body :namespace)
       decode-graph
       (graph/update-graph! namespace))
  (graph/graph namespace))

(api/define-routes)

(ns metabase.api.common
  (:require
   [clojure.set :as set]
   [clojure.spec.alpha :as s]
   [clojure.string :as str]
   [compojure.core :as compojure]
   [medley.core :as m]
   [metabase.api.common.internal
    :refer [add-route-param-schema
            auto-coerce
            route-dox
            validate-params
            route-fn-name
            wrap-response-if-needed]]
   [metabase.config :as config]
   [metabase.events :as events]
   [metabase.models.interface :as mi]
   [metabase.util :as u]
   [metabase.util.i18n :as i18n :refer [deferred-tru tru]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [ring.middleware.multipart-params :as mp]
   [toucan2.core :as t2]))

(declare check-403 check-404)

(def ^:dynamic ^Integer *current-user-id*
  nil)

(def ^:dynamic *current-user*
  (atom nil)) ; default binding is just something that will return nil when dereferenced

(def ^:dynamic ^Boolean *is-superuser?*
  false)

(def ^:dynamic ^Boolean *is-group-manager?*
  false)

(def ^:dynamic *current-user-permissions-set*
  (atom #{}))

(defn- check-one [condition code message]
  (when-not condition
    (let [[message info] (if (and (map? message)
                                  (not (i18n/localized-string? message)))
                           [(:message message) message]
                           [message])]
      (throw (ex-info (str message) (assoc info :status-code code)))))
  condition)

(defn check
  {:style/indent 1, :arglists '([condition [code message] & more] [condition code message & more])}
  [condition & args]
  (let [[code message & more] (if (sequential? (first args))
                                (concat (first args) (rest args))
                                args)]
    (check-one condition code message)
    (if (seq more)
      (recur (first more) (rest more))
      condition)))

(defn check-exists?
  ([entity id]
   (check-exists? entity :id id))
  ([entity k v & more]
   (check-404 (apply t2/exists? entity k v more))))

(defn check-superuser
  []
  (check-403 *is-superuser?*))

(defn throw-invalid-param-exception
  [field-name message]
  (throw (ex-info (tru "Invalid field: {0}" field-name)
           {:status-code 400
            :errors      {(keyword field-name) message}})))

(defn checkp
  {:style/indent 1}
  ([tst field-name message]
   (when-not tst
     (throw-invalid-param-exception (str field-name) message))))

(defmacro do-api-let
  [response-pair bindings & body]
  ;; so `response-pair` doesn't get evaluated more than once
  (let [response-pair-symb (gensym "response-pair-")]
    `(let [~response-pair-symb ~response-pair
           ~@(vec (apply concat (for [[binding test] (partition-all 2 bindings)]
                                  [binding `(check ~test ~response-pair-symb)])))]
       ~@body)))

(def ^:private generic-400
  [400 (deferred-tru "Invalid Request.")])

(defn check-400
  [arg]
  (check arg generic-400))

(def ^:private generic-404
  [404 (deferred-tru "Not found.")])

(defn check-404
  [arg]
  (check arg generic-404))

(defmacro let-404
  {:style/indent 1}
  [bindings & body]
  `(do-api-let ~generic-404 ~bindings ~@body))

(defn- generic-403 []
  [403 (tru "You don''t have permissions to do that.")])

(defn check-403
  [arg]
  (check arg (generic-403)))

(defn throw-403
  ([]
   (throw-403 nil))
  ([e]
   (throw (ex-info (tru "You don''t have permissions to do that.") {:status-code 403} e))))

(def ^:private generic-500
  [500 (deferred-tru "Internal server error.")])

(defn check-500
  [arg]
  (check arg generic-500))

(def generic-204-no-content
  {:status 204, :body nil})

(s/def ::defendpoint-args
  (s/cat
   :method      symbol?
   :route       (some-fn string? sequential?)
   :docstr      (s/? string?)
   :args        vector?
   :arg->schema (s/? (s/map-of symbol? any?)) ;; any? is either a plumatic or malli schema
   :body        (s/* any?)))

(defn- parse-defendpoint-args [args]
  (let [parsed (s/conform ::defendpoint-args args)]
    (when (= parsed ::s/invalid)
      (throw (ex-info (str "Invalid defendpoint args: " (s/explain-str ::defendpoint-args args))
                      (s/explain-data ::defendpoint-args args))))
    (let [{:keys [method route docstr args arg->schema body]} parsed
          fn-name                                             (route-fn-name method route)
          route                                               (add-route-param-schema arg->schema route)
          ;; eval the vals in arg->schema to make sure the actual schemas are resolved so we can document
          ;; their API error messages
          docstr                                              (route-dox method route docstr args
                                                                               (m/map-vals #_{:clj-kondo/ignore [:discouraged-var]} eval arg->schema)
                                                                               body)]
      ;; Don't i18n this, it's dev-facing only
      (when-not docstr
        (log/warn (u/format-color 'red "Warning: endpoint %s/%s does not have a docstring. Go add one."
                                  (ns-name *ns*) fn-name)))
      (assoc parsed :fn-name fn-name, :route route, :docstr docstr))))

(defn validate-param-values
  [{method :request-method uri :uri body :body} expected-params]
  (when (and (not config/is-prod?)
             (map? body))
    (let [extraneous-params (set/difference (set (keys body))
                                            (set expected-params))]
      (when (seq extraneous-params)
        (log/warnf "Unexpected parameters at %s: %s\nPlease add them to the schema or remove them from the API client"
                   [method uri] (vec extraneous-params))))))

(defn method-symbol->keyword
  [method-symbol]
  (-> method-symbol
      name
      u/lower-case-en
      keyword))

(defmacro defendpoint*
  [{:keys [method route fn-name docstr args body arg->schema]}]
  {:pre [(or (string? route) (vector? route))]}
  (let [method-kw       (method-symbol->keyword method)
        allowed-params  (mapv keyword (keys arg->schema))
        prep-route      #'compojure/prepare-route
        multipart?      (get (meta method) :multipart false)
        handler-wrapper (if multipart? mp/wrap-multipart-params identity)]
    `(def ~(vary-meta fn-name
                      assoc
                      :doc          docstr
                      :is-endpoint? true)
       ;; The next form is a copy of `compojure/compile-route`, with the sole addition of the call to
       ;; `validate-param-values`. This is because to validate the request body we need to intercept the request
       ;; before the destructuring takes place. I.e., we need to validate the value of `(:body request#)`, and that's
       ;; not available if we called `compile-route` ourselves.
       (compojure/make-route
        ~method-kw
        ~(prep-route route)
        (~handler-wrapper
         (fn [request#]
           (validate-param-values request# (quote ~allowed-params))
           (compojure/let-request [~args request#]
                                  ~@body)))))))

(defmacro defendpoint
  {:arglists '([method route docstr? args schemas-map? & body])}
  [& defendpoint-args]
  (let [{:keys [args body arg->schema], :as defendpoint-args} (parse-defendpoint-args defendpoint-args)]
    `(defendpoint* ~(assoc defendpoint-args
                           :body `((auto-coerce ~args ~arg->schema
                                                ~@(validate-params arg->schema)
                                                (wrap-response-if-needed
                                                 (do ~@body))))))))

(defmacro defendpoint-async
  {:arglists '([method route docstr? args schemas-map? & body])}
  [& defendpoint-args]
  (let [{:keys [args body arg->schema], :as defendpoint-args} (parse-defendpoint-args defendpoint-args)]
    `(defendpoint* ~(assoc defendpoint-args
                           :args []
                           :body `((fn ~args
                                     ~@(validate-params arg->schema)
                                     ~@body))))))

(defn- namespace->api-route-fns
  [nmspace]
  (for [[_symb varr] (ns-publics nmspace)
        :when       (:is-endpoint? (meta varr))]
    varr))

(defn- api-routes-docstring [nmspace route-fns middleware]
  (str
   (format "Ring routes for %s:\n%s"
           (-> (ns-name nmspace)
               (str/replace #"^metabase\." )
               (str/replace #"\." "/"))
           (u/pprint-to-str route-fns))
   (when (seq middleware)
     (str "\nMiddleware applied to all endpoints in this namespace:\n"
          (u/pprint-to-str middleware)))))

(defmacro define-routes
  {:style/indent 0}
  [& middleware]
  (let [api-route-fns (namespace->api-route-fns *ns*)
        routes        `(compojure/routes ~@api-route-fns)
        docstring     (str "Routes for " *ns*)]
    `(def ~(vary-meta 'routes assoc :doc (api-routes-docstring *ns* api-route-fns middleware))
       ~docstring
       ~(if (seq middleware)
          `(-> ~routes ~@middleware)
          routes))))

(defn +check-superuser
  [handler]
  (fn
    ([request]
     (check-superuser)
     (handler request))
    ([request respond raise]
     (if-let [e (try
                  (check-superuser)
                  nil
                  (catch Throwable e
                    e))]
       (raise e)
       (handler request respond raise)))))

(defn read-check
  {:style/indent 2}
  ([obj]
   (check-404 obj)
   (try
     (check-403 (mi/can-read? obj))
     (catch clojure.lang.ExceptionInfo e
       (events/publish-event! :event/read-permission-failure {:user-id    *current-user-id*
                                                              :object     obj
                                                              :has-access false})
       (throw e)))
   obj)
  ([entity id]
   (read-check (t2/select-one entity :id id)))
  ([entity id & other-conditions]
   (read-check (apply t2/select-one entity :id id other-conditions))))

(defn write-check
  {:style/indent 2}
  ([obj]
   (check-404 obj)
   (try
     (check-403 (mi/can-write? obj))
     (catch clojure.lang.ExceptionInfo e
       (events/publish-event! :event/write-permission-failure {:user-id *current-user-id*
                                                               :object obj})
       (throw e)))
   obj)
  ([entity id]
   (write-check (t2/select-one entity :id id)))
  ([entity id & other-conditions]
   (write-check (apply t2/select-one entity :id id other-conditions))))

(defn create-check
  {:added "0.32.0", :style/indent 2}
  [entity m]
  (try
    (check-403 (mi/can-create? entity m))
    (catch clojure.lang.ExceptionInfo e
      (events/publish-event! :event/create-permission-failure {:model entity
                                                               :user-id *current-user-id*})
      (throw e))))

(defn update-check
  {:added "0.36.0", :style/indent 2}
  [instance changes]
  (try
    (check-403 (mi/can-update? instance changes))
    (catch clojure.lang.ExceptionInfo e
      (events/publish-event! :event/update-permission-failure {:user-id *current-user-id*
                                                               :object instance})
      (throw e))))

(defn check-not-archived
  [object]
  (u/prog1 object
    (check-404 object)
    (check (not (:archived object))
      [404 {:message (tru "The object has been archived."), :error_code "archived"}])))

(defn check-valid-page-params
  [limit offset]
  (check (not (and limit (not offset))) [400 (tru "When including a limit, an offset must also be included.")])
  (check (not (and offset (not limit))) [400 (tru "When including an offset, a limit must also be included.")]))

(mu/defn column-will-change? :- :boolean
  "Helper for PATCH-style operations to see if a column is set to change when `object-updates` (i.e., the input to the
  endpoint) is applied.
    ;; assuming we have a Collection 10, that is not currently archived...
    (api/column-will-change? :archived (t2/select-one Collection :id 10) {:archived true}) ; -> true, because value will change
    (api/column-will-change? :archived (t2/select-one Collection :id 10) {:archived false}) ; -> false, because value did not change
    (api/column-will-change? :archived (t2/select-one Collection :id 10) {}) ; -> false; value not specified in updates (request body)"
  [k :- :keyword object-before-updates :- :map object-updates :- :map]
  (boolean
   (and (contains? object-updates k)
        (not= (get object-before-updates k)
              (get object-updates k)))))

(mu/defn reconcile-position-for-collection!
  [collection-id :- [:maybe ms/PositiveInt]
   old-position  :- [:maybe ms/PositiveInt]
   new-position  :- [:maybe ms/PositiveInt]]
  (let [update-fn! (fn [plus-or-minus position-update-clause]
                     (doseq [model '[Card Dashboard Pulse]]
                       (t2/update! model {:collection_id       collection-id
                                          :collection_position position-update-clause}
                                   {:collection_position [plus-or-minus :collection_position 1]})))]
    (when (not= new-position old-position)
      (cond
        (and (nil? new-position)
             old-position)
        (update-fn! :-  [:> old-position])
        (and new-position (nil? old-position))
        (update-fn! :+ [:>= new-position])
        (> new-position old-position)
        (update-fn! :- [:between old-position new-position])
        (< new-position old-position)
        (update-fn! :+ [:between new-position old-position])))))

(def ^:private ModelWithPosition
  [:map
   [:collection_id       [:maybe ms/PositiveInt]]
   [:collection_position [:maybe ms/PositiveInt]]])

(def ^:private ModelWithOptionalPosition
  [:map
   [:collection_id       {:optional true} [:maybe ms/PositiveInt]]
   [:collection_position {:optional true} [:maybe ms/PositiveInt]]])

(mu/defn maybe-reconcile-collection-position!
  ([new-model-data :- ModelWithPosition]
   (maybe-reconcile-collection-position! nil new-model-data))
  ([{old-collection-id :collection_id, old-position :collection_position, :as _before-update} :- [:maybe ModelWithPosition]
    {new-collection-id :collection_id, new-position :collection_position, :as model-updates} :- ModelWithOptionalPosition]
   (let [updated-collection? (and (contains? model-updates :collection_id)
                                  (not= old-collection-id new-collection-id))
         updated-position?   (and (contains? model-updates :collection_position)
                                  (not= old-position new-position))]
     (cond
       ;; If the collection hasn't changed, but we have a new collection position, we might need to reconcile
       (and (not updated-collection?) updated-position?)
       (reconcile-position-for-collection! old-collection-id old-position new-position)
       ;; If we have a new collection id, but no new position, reconcile the old collection, then update the new
       ;; collection with the existing position
       (and updated-collection? (not updated-position?))
       (do
         (reconcile-position-for-collection! old-collection-id old-position nil)
         (reconcile-position-for-collection! new-collection-id nil old-position))
       ;; We have a new collection id AND and new collection position
       ;; Update the old collection using the old position
       ;; Update the new collection using the new position
       (and updated-collection? updated-position?)
       (do
         (reconcile-position-for-collection! old-collection-id old-position nil)
         (reconcile-position-for-collection! new-collection-id nil new-position))))))

(defn bit->boolean
  [v]
  (if (number? v)
    (not (zero? v))
    v))

(defn parse-multi-values-param
  [xs parse-fn]
  (if (sequential? xs)
    (map parse-fn xs)
    [(parse-fn xs)]))

(ns metabase.api.common.internal
  (:require
   [clojure.string :as str]
   [clojure.walk :as walk]
   [colorize.core :as colorize]
   [malli.core :as mc]
   [malli.error :as me]
   [malli.transform :as mtx]
   [metabase.async.streaming-response :as streaming-response]
   [metabase.config :as config]
   [metabase.util :as u]
   [metabase.util.i18n :refer [tru]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.describe :as umd]
   [metabase.util.malli.schema :as ms]
   [potemkin.types :as p.types])
  (:import
   (metabase.async.streaming_response StreamingResponse)))

(set! *warn-on-reflection* true)

(comment streaming-response/keep-me)

(defn- endpoint-name
  ([method route]
   (endpoint-name *ns* method route))
  ([endpoint-namespace method route]
   (format "%s %s%s"
           (name method)
           (-> (.getName (the-ns endpoint-namespace))
               (str/replace #"^metabase\.api\." "/api/")
               ;; HACK to make sure some enterprise endpoints are consistent with the code.
               ;; The right way to fix this is to move them -- see #22687
               ;; /api/ee/sandbox/table -> /api/table, this is an override route for /api/table if sandbox is available
               (str/replace #"^metabase-enterprise\.sandbox\.api\.table" "/api/table")
               ;; /api/ee/sandbox -> /api/mt
               (str/replace #"^metabase-enterprise\.sandbox\.api\." "/api/mt/")
               ;; /api/ee/content-verification -> /api/moderation-review
               (str/replace #"^metabase-enterprise\.content-verification\.api\." "/api/moderation-review/")
               ;; /api/ee/sso/sso/ -> /auth/sso
               (str/replace #"^metabase-enterprise\.sso\.api\." "/auth/")
               ;; this should be only the replace for enterprise once we resolved #22687
               (str/replace #"^metabase-enterprise\.([^\.]+)\.api\." "/api/ee/$1/"))
           (if (vector? route)
             (first route)
             route))))

(defn- args-form-flatten
  [form]
  (cond
    (map? form) (args-form-flatten (mapcat (fn [[k v]]
                                             [(args-form-flatten k) (args-form-flatten v)])
                                           form))
    (sequential? form) (mapcat args-form-flatten form)
    :else       [form]))

(defn- args-form-symbols
  [form]
  (into {} (for [arg   (args-form-flatten form)
                 :when (and (symbol? arg)
                            (not= arg 'body))]
             {arg nil})))

(defn- dox-for-schema
  [schema route-str]
  (try (umd/describe schema)
       (catch Exception _
         (ex-data
          (when (and schema config/is-dev?) ;; schema is nil for any var without a schema. That's ok!
            (log/warn
             (u/format-color 'red (str "Invalid Malli Schema: %s defined at %s")
                             (u/pprint-to-str schema)
                             (u/add-period route-str)))))
         "")))

(defn- param-name
  [param-symb schema]
  (or (when (record? schema)
        (:api-param-name schema))
      (name param-symb)))

(defn- format-route-schema-dox
  [param-symb->schema route-str]
  ;; these are here
  (when (seq param-symb->schema)
    (str "\n\n### PARAMS:\n\n"
         (str/join "\n\n"
                   (for [[param-symb schema] param-symb->schema]
                     (format "*  **`%s`** %s"
                             (param-name param-symb schema)
                             (dox-for-schema schema route-str)))))))

(defn- format-route-dox
  [route-str docstr param->schema]
  (str (format "## `%s`" route-str)
       (when (seq docstr)
         (str "\n\n" (u/add-period docstr)))
       (format-route-schema-dox param->schema route-str)))

(defn- contains-superuser-check?
  [body]
  (let [body (set body)]
    (or (contains? body '(check-superuser))
        (contains? body '(api/check-superuser)))))

(defn route-dox
  [method route docstr args param->schema body]
  (format-route-dox (endpoint-name method route)
                    (str (u/add-period docstr) (when (contains-superuser-check? body)
                                                 "\n\nYou must be a superuser to do this."))
                    (merge (args-form-symbols args)
                           param->schema)))

(defn parse-int
  [^String value]
  (try (Integer/parseInt value)
       (catch NumberFormatException _
         (throw (ex-info (tru "Not a valid integer: ''{0}''" value) {:status-code 400})))))

(def ^:dynamic *auto-parse-types*
  {:int  {:route-param-regex #"[0-9]+"
          :parser            'metabase.api.common.internal/parse-int}
   :uuid {:route-param-regex u/uuid-regex
          :parser            nil}})

(def ^:private ^:const  auto-parse-arg-name-patterns
  [[#"^uuid$"       :uuid]
   [#"^session_id$" :uuid]
   [#"^[\w-_]*id$"  :int]])

(defn arg-type
  [arg]
  (some (fn [[pattern type]]
          (when (re-find pattern (name arg))
            type))
        auto-parse-arg-name-patterns))

(defn route-param-regex
  [arg]
  (some->> (arg-type arg)
           *auto-parse-types*
           :route-param-regex
           (vector arg)))

(defn route-arg-keywords
  [route]
  (->> (re-seq #":([\w-]+)" route)
       (map second)
       (map keyword)))

(defn- ->matching-regex
  [schema]
  (let [schema-type (try (mc/type schema)
                         (catch clojure.lang.ExceptionInfo _
                           (mc/type #_:clj-kondo/ignore (eval schema))))]
    [schema-type
     (condp = schema-type
       ;; can use any regex directly
       :re (first (try (mc/children schema)
                       (catch clojure.lang.ExceptionInfo _
                         (mc/children #_:clj-kondo/ignore (eval schema)))))
       :keyword #"[\S]+"
       'pos-int? #"[0-9]+"
       :int #"-?[0-9]+"
       'int? #"-?[0-9]+"
       :uuid u/uuid-regex
       'uuid? u/uuid-regex
       nil)]))

(def ^:private no-regex-schemas #{(mc/type ms/NonBlankString)
                                  (mc/type (mc/schema [:maybe ms/PositiveInt]))
                                  (mc/type [:enum "a" "b"])
                                  :fn
                                  :string})

(defn add-route-param-schema
  [arg->schema route]
  (if (vector? route)
    route
    (let [[wildcard & wildcards]
          (->> (for [[k schema] arg->schema
                     :when      (re-find (re-pattern (str ":" k)) route)
                     :let       [[schema-type re] (->matching-regex schema)]]
                 (if re
                   [route (keyword k) re]
                   (when (and config/is-dev? (not (contains? no-regex-schemas schema-type)))
                     (let [overview (str "Warning: missing route-param regex for schema: "
                                         route " " [k schema])
                           fix      (str "Either add `" (pr-str schema-type) "` to "
                                         "metabase.api.common.internal/->matching-regex or "
                                         "metabase.api.common.internal/no-regex-schemas.")]
                       (log/warn (colorize/red overview))
                       (log/warn (colorize/green fix))))))
               (remove nil?))]
      (cond
        ;; multiple hits -> tack them onto the original route shape.
        wildcards (vec (reduce into wildcard (mapv #(drop 1 %) wildcards)))
        wildcard  wildcard
        :else     route))))

(defn let-form-for-arg
  [arg-symbol]
  (when (symbol? arg-symbol)
    (some-> (arg-type arg-symbol)                                     ; :int
            *auto-parse-types*                                        ; {:parser ... }
            :parser                                                   ; Integer/parseInt
            ((fn [parser] `(when ~arg-symbol (~parser ~arg-symbol)))) ; (when id (Integer/parseInt id))
            ((partial vector arg-symbol)))))                          ; [id (Integer/parseInt id)]

(defmacro auto-parse
  {:style/indent 1}
  [args & body]
  (let [let-forms (->> args
                       (mapcat let-form-for-arg)
                       (filter identity))]
    `(let [~@let-forms]
       ~@body)))

(def defendpoint-transformer
  (mtx/transformer
   (mtx/string-transformer)
   (mtx/json-transformer)))

(defn- extract-symbols [in]
  (let [*symbols (atom [])]
    (walk/postwalk
     (fn [x] (when (symbol? x) (swap! *symbols conj x)) x)
     in)
    @*symbols))

(defn- mauto-let-form [arg->schema arg-symbol]
  (when arg->schema
    (when-let [schema (arg->schema arg-symbol)]
      `[~arg-symbol (mc/decode ~schema ~arg-symbol defendpoint-transformer)])))

(defmacro auto-coerce
  {:style/indent 1}
  [args arg->schema & body]
  (let [let-forms (->> args
                       extract-symbols
                       (mapcat #(mauto-let-form arg->schema %))
                       (remove nil?))]
    `(let [~@let-forms] ~@body)))

(defn validate-param
  [field-name value schema]
  (when-not (mc/validate schema value)
    (throw (ex-info (tru "Invalid m field: {0}" field-name)
                    {:status-code 400
                     :errors      {(keyword field-name) (umd/describe schema)}
                     :specific-errors {(keyword field-name)
                                       (-> schema
                                           (mc/explain value)
                                           me/with-spell-checking
                                           (me/humanize {:wrap mu/humanize-include-value}))}}))))

(defn validate-params
  [param->schema]
  (for [[param schema] param->schema]
    `(validate-param '~param ~param ~schema)))

(defn route-fn-name
  [method route]
  ;; if we were passed a vector like [":id" :id #"[0-9+]"] only use first part
  (let [route (if (vector? route) (first route) route)]
    (-> (str (name method) route)
        (^String .replace "/" "_")
        symbol)))

(p.types/defprotocol+ EndpointResponse
  (wrap-response-if-needed [this]
    "Transform the value returned by a `defendpoint` form as needed, e.g. by adding `:status` and `:body`."))

(extend-protocol EndpointResponse
  Object
  (wrap-response-if-needed [this]
    {:status 200, :body this})

  nil
  (wrap-response-if-needed [_]
    {:status 204, :body nil})

  StreamingResponse
  (wrap-response-if-needed [this]
    this)

  clojure.lang.IPersistentMap
  (wrap-response-if-needed [m]
    (if (and (:status m) (contains? m :body))
      m
      {:status 200, :body m})))

(ns metabase.api.common.validation
  (:require
   [clojure.string :as str]
   [metabase.api.common :as api]
   [metabase.config :as config]
   [metabase.embed.settings :as embed.settings]
   [metabase.plugins.classloader :as classloader]
   [metabase.public-settings :as public-settings]
   [metabase.public-settings.premium-features :as premium-features]
   [metabase.util.i18n :refer [tru]]))

(defn check-public-sharing-enabled
  []
  (api/check (public-settings/enable-public-sharing)
             [400 (tru "Public sharing is not enabled.")]))

(defn check-embedding-enabled
  []
  (api/check (embed.settings/enable-embedding)
             [400 (tru "Embedding is not enabled.")]))

(defn check-has-application-permission
  ([perm-type]
   (check-has-application-permission perm-type true))
  ([perm-type require-superuser?]
   (if-let [f (and (premium-features/enable-advanced-permissions?)
                   (resolve 'metabase-enterprise.advanced-permissions.common/current-user-has-application-permissions?))]
     (api/check-403 (f perm-type))
     (when require-superuser?
       (api/check-superuser)))))

(defn check-advanced-permissions-enabled
  [perm-type]
  (api/check (premium-features/enable-advanced-permissions?)
             [402 (tru "The {0} permissions functionality is only enabled if you have a premium token with the advanced-permissions feature."
                       (str/replace (name perm-type) "-" " "))]))

(defn check-group-manager
  ([]
   (check-group-manager true))
  ([require-superuser?]
   (if (premium-features/enable-advanced-permissions?)
     (api/check-403 (or api/*is-superuser?* api/*is-group-manager?*))
     (when require-superuser?
       (api/check-superuser)))))

(defn check-manager-of-group
  ([group-or-id]
   (check-manager-of-group group-or-id true))
  ([group-or-id require-superuser?]
   (when config/ee-available?
     (classloader/require 'metabase-enterprise.advanced-permissions.common))
   (if-let [f (and (premium-features/enable-advanced-permissions?)
                   (resolve 'metabase-enterprise.advanced-permissions.common/current-user-is-manager-of-group?))]
     (api/check-403 (or api/*is-superuser?* (f group-or-id)))
     (when require-superuser?
       (api/check-superuser)))))

(ns metabase.api.dashboard
  (:require
   [cheshire.core :as json]
   [clojure.set :as set]
   [compojure.core :refer [DELETE GET POST PUT]]
   [medley.core :as m]
   [metabase.actions.execution :as actions.execution]
   [metabase.analytics.snowplow :as snowplow]
   [metabase.api.common :as api]
   [metabase.api.common.validation :as validation]
   [metabase.api.dataset :as api.dataset]
   [metabase.automagic-dashboards.populate :as populate]
   [metabase.events :as events]
   [metabase.mbql.normalize :as mbql.normalize]
   [metabase.mbql.schema :as mbql.s]
   [metabase.mbql.util :as mbql.u]
   [metabase.models.action :as action]
   [metabase.models.card :as card :refer [Card]]
   [metabase.models.collection :as collection]
   [metabase.models.collection.root :as collection.root]
   [metabase.models.dashboard :as dashboard :refer [Dashboard]]
   [metabase.models.dashboard-card :as dashboard-card :refer [DashboardCard]]
   [metabase.models.dashboard-tab :as dashboard-tab]
   [metabase.models.field :refer [Field]]
   [metabase.models.interface :as mi]
   [metabase.models.params :as params]
   [metabase.models.params.chain-filter :as chain-filter]
   [metabase.models.params.custom-values :as custom-values]
   [metabase.models.query :as query :refer [Query]]
   [metabase.models.query.permissions :as query-perms]
   [metabase.models.revision :as revision]
   [metabase.models.revision.last-edit :as last-edit]
   [metabase.models.table :refer [Table]]
   [metabase.query-processor.dashboard :as qp.dashboard]
   [metabase.query-processor.error-type :as qp.error-type]
   [metabase.query-processor.middleware.constraints :as qp.constraints]
   [metabase.query-processor.middleware.permissions :as qp.perms]
   [metabase.query-processor.pivot :as qp.pivot]
   [metabase.query-processor.util :as qp.util]
   [metabase.related :as related]
   [metabase.util :as u]
   [metabase.util.i18n :refer [deferred-tru tru]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [steffan-westcott.clj-otel.api.trace.span :as span]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(defn- hydrate-dashboard-details
  [{dashboard-id :id :as dashboard}]
  ;; I'm a bit worried that this is an n+1 situation here. The cards can be batch hydrated i think because they
  ;; have a hydration key and an id. moderation_reviews currently aren't batch hydrated but i'm worried they
  ;; cannot be in this situation
  (span/with-span!
    {:name       "hydrate-dashboard-details"
     :attributes {:dashboard/id dashboard-id}}
    (t2/hydrate dashboard [:dashcards
                           [:card [:moderation_reviews :moderator_details]]
                           [:card :can_write]
                           :series
                           :dashcard/action
                           :dashcard/linkcard-info]
                :tabs
                :collection_authority_level
                :can_write
                :param_fields
                :param_values
                [:collection :is_personal])))

(api/defendpoint POST 
  "Create a new Dashboard."
  [:as {{:keys [name description parameters cache_ttl collection_id collection_position], :as _dashboard} :body}]
  {name                ms/NonBlankString
   parameters          [:maybe [:sequential ms/Parameter]]
   description         [:maybe :string]
   cache_ttl           [:maybe ms/PositiveInt]
   collection_id       [:maybe ms/PositiveInt]
   collection_position [:maybe ms/PositiveInt]}
  ;; if we're trying to save the new dashboard in a Collection make sure we have permissions to do that
  (collection/check-write-perms-for-collection collection_id)
  (let [dashboard-data {:name                name
                        :description         description
                        :parameters          (or parameters [])
                        :creator_id          api/*current-user-id*
                        :cache_ttl           cache_ttl
                        :collection_id       collection_id
                        :collection_position collection_position}
        dash           (t2/with-transaction [_conn]
                        ;; Adding a new dashboard at `collection_position` could cause other dashboards in this collection to change
                        ;; position, check that and fix up if needed
                        (api/maybe-reconcile-collection-position! dashboard-data)
                        ;; Ok, now save the Dashboard
                        (first (t2/insert-returning-instances! :model/Dashboard dashboard-data)))]
    (events/publish-event! :event/dashboard-create {:object dash :user-id api/*current-user-id*})
    (snowplow/track-event! ::snowplow/dashboard-created api/*current-user-id* {:dashboard-id (u/the-id dash)})
    (-> dash
        hydrate-dashboard-details
        collection.root/hydrate-root-collection
        (assoc :last-edit-info (last-edit/edit-information-for-user @api/*current-user*)))))

(defn- hide-unreadable-card
  [card]
  (when card
    (if (mi/can-read? card)
      card
      (select-keys card [:id]))))

(defn- hide-unreadable-cards
  [dashboard]
  (update dashboard :dashcards (fn [dashcards]
                                 (vec (for [dashcard dashcards]
                                        (-> dashcard
                                            (update :card hide-unreadable-card)
                                            (update :series (partial mapv hide-unreadable-card))))))))

(defn- card->query-hashes
  [{:keys [dataset_query]}]
  (u/ignore-exceptions
    [(qp.util/query-hash dataset_query)
     (qp.util/query-hash (assoc dataset_query :constraints (qp.constraints/default-query-constraints)))]))

(defn- dashcard->query-hashes
  [{:keys [card series]}]
  (reduce concat
          (card->query-hashes card)
          (for [card series]
            (card->query-hashes card))))

(defn- dashcards->query-hashes
  [dashcards]
  (apply concat (for [dashcard dashcards]
                  (dashcard->query-hashes dashcard))))

(defn- hashes->hash-vec->avg-time
  [hashes]
  (when (seq hashes)
    (into {} (for [[k v] (t2/select-fn->fn :query_hash :average_execution_time Query :query_hash [:in hashes])]
               {(vec k) v}))))

(defn- add-query-average-duration-to-card
  [card hash-vec->avg-time]
  (assoc card :query_average_duration (some (fn [query-hash]
                                              (hash-vec->avg-time (vec query-hash)))
                                            (card->query-hashes card))))

(defn- add-query-average-duration-to-dashcards
  ([dashcards]
   (add-query-average-duration-to-dashcards dashcards (hashes->hash-vec->avg-time (dashcards->query-hashes dashcards))))
  ([dashcards hash-vec->avg-time]
   (for [dashcard dashcards]
     (-> dashcard
         (update :card   add-query-average-duration-to-card hash-vec->avg-time)
         (update :series (fn [series]
                           (for [card series]
                             (add-query-average-duration-to-card card hash-vec->avg-time))))))))

(defn add-query-average-durations
  [dashboard]
  (update dashboard :dashcards add-query-average-duration-to-dashcards))

(defn- get-dashboard
  [id]
  (span/with-span!
    {:name       "get-dashboard"
     :attributes {:dashboard/id id}}
    (-> (t2/select-one :model/Dashboard :id id)
        api/read-check
        hydrate-dashboard-details
        collection.root/hydrate-root-collection
        api/check-not-archived
        hide-unreadable-cards
        add-query-average-durations)))

(defn- cards-to-copy
  [dashcards]
  (letfn [(split-cards [{:keys [card series] :as db-card}]
            (cond
              (nil? (:card_id db-card)) ; text card
              {}
              ;; cards without permissions are just a map with an :id from [[hide-unreadable-card]]
              (not (mi/model card))
              {:retain nil, :discard (into [card] series)}
              (mi/can-read? card)
              (let [{writable true unwritable false} (group-by (comp boolean mi/can-read?)
                                                               series)]
                {:retain (into [card] writable), :discard unwritable})
              ;; if you can't write the base, we don't have anywhere to put the series
              :else
              {:discard (into [card] series)}))]
    (reduce (fn [acc db-card]
              (let [{:keys [retain discard]} (split-cards db-card)]
                (-> acc
                    (update :copy merge (m/index-by :id retain))
                    (update :discard concat discard))))
            {:copy {}
             :discard []}
            dashcards)))

(defn- duplicate-cards
  [dashboard dest-coll-id]
  (let [same-collection? (= (:collection_id dashboard) dest-coll-id)
        {:keys [copy discard]} (cards-to-copy (:dashcards dashboard))]
    (reduce (fn [m [id card]]
              (assoc-in m
                        [:copied id]
                        (if (:dataset card)
                          card
                          (card/create-card!
                           (cond-> (assoc card :collection_id dest-coll-id)
                             same-collection?
                             (update :name #(str % " - " (tru "Duplicate"))))
                           @api/*current-user*
                           ;; creating cards from a transaction. wait until tx complete to signal event
                           true))))
            {:copied {}
             :uncopied discard}
            copy)))

(defn- duplicate-tabs
  [new-dashboard existing-tabs]
  (let [new-tab-ids (t2/insert-returning-pks! :model/DashboardTab
                                              (for [tab existing-tabs]
                                                (-> tab
                                                    (assoc :dashboard_id (:id new-dashboard))
                                                    (dissoc :id :entity_id :created_at :updated_at))))]
    (zipmap (map :id existing-tabs) new-tab-ids)))

(defn update-cards-for-copy
  [dashboard-id dashcards deep? id->new-card id->new-tab-id]
  (when (and deep? (nil? id->new-card))
    (throw (ex-info (tru "No copied card information found")
                    {:user-id api/*current-user-id*
                     :dashboard-id dashboard-id})))
  (let [dashcards (if (seq id->new-tab-id)
                    (map #(assoc % :dashboard_tab_id (id->new-tab-id (:dashboard_tab_id %)))
                         dashcards)
                    dashcards)]
    (if-not deep?
      dashcards
      (keep (fn [dashboard-card]
              (cond
               ;; text cards need no manipulation
               (nil? (:card_id dashboard-card))
               dashboard-card
               ;; if we didn't duplicate, it doesn't go in the dashboard
               (not (id->new-card (:card_id dashboard-card)))
               nil
               :else
               (let [new-id (fn [id]
                              (-> id id->new-card :id))]
                 (-> dashboard-card
                     (update :card_id new-id)
                     (assoc :card (-> dashboard-card :card_id id->new-card))
                     (m/update-existing :parameter_mappings
                                        (fn [pms]
                                          (keep (fn [pm]
                                                  (m/update-existing pm :card_id new-id))
                                                pms)))
                     (m/update-existing :series
                                        (fn [series]
                                          (keep (fn [card]
                                                  (when-let [id' (new-id (:id card))]
                                                    (assoc card :id id')))
                                                series)))))))
            dashcards))))

(api/defendpoint POST 
  "Copy a Dashboard."
  [from-dashboard-id :as {{:keys [name description collection_id collection_position
                                  is_deep_copy], :as _dashboard} :body}]
  {from-dashboard-id      [:maybe ms/PositiveInt]
   name                   [:maybe ms/NonBlankString]
   description            [:maybe :string]
   collection_id          [:maybe ms/PositiveInt]
   collection_position    [:maybe ms/PositiveInt]
   is_deep_copy           [:maybe :boolean]}
  ;; if we're trying to save the new dashboard in a Collection make sure we have permissions to do that
  (collection/check-write-perms-for-collection collection_id)
  (let [existing-dashboard (get-dashboard from-dashboard-id)
        dashboard-data {:name                (or name (:name existing-dashboard))
                        :description         (or description (:description existing-dashboard))
                        :parameters          (or (:parameters existing-dashboard) [])
                        :creator_id          api/*current-user-id*
                        :collection_id       collection_id
                        :collection_position collection_position}
        new-cards      (atom nil)
        dashboard      (t2/with-transaction [_conn]
                        ;; Adding a new dashboard at `collection_position` could cause other dashboards in this
                        ;; collection to change position, check that and fix up if needed
                        (api/maybe-reconcile-collection-position! dashboard-data)
                        ;; Ok, now save the Dashboard
                        (let [dash (first (t2/insert-returning-instances! :model/Dashboard dashboard-data))
                              {id->new-card :copied uncopied :uncopied}
                              (when is_deep_copy
                                (duplicate-cards existing-dashboard collection_id))
                              id->new-tab-id (when-let [existing-tabs (seq (:tabs existing-dashboard))]
                                               (duplicate-tabs dash existing-tabs))]
                          (reset! new-cards (vals id->new-card))
                          (when-let [dashcards (seq (update-cards-for-copy from-dashboard-id
                                                                           (:dashcards existing-dashboard)
                                                                           is_deep_copy
                                                                           id->new-card
                                                                           id->new-tab-id))]
                            (api/check-500 (dashboard/add-dashcards! dash dashcards)))
                          (cond-> dash
                            (seq uncopied)
                            (assoc :uncopied uncopied))))]
    (snowplow/track-event! ::snowplow/dashboard-created api/*current-user-id* {:dashboard-id (u/the-id dashboard)})
    ;; must signal event outside of tx so cards are visible from other threads
    (when-let [newly-created-cards (seq @new-cards)]
      (doseq [card newly-created-cards]
        (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})))
    (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})
    dashboard))

(api/defendpoint GET 
  "Get Dashboard with ID."
  [id]
  {id ms/PositiveInt}
  (let [dashboard (get-dashboard id)]
    (events/publish-event! :event/dashboard-read {:object dashboard :user-id api/*current-user-id*})
    (last-edit/with-last-edit-info dashboard :dashboard)))

(defn- check-allowed-to-change-embedding
  [dash-before-update dash-updates]
  (when (or (api/column-will-change? :enable_embedding dash-before-update dash-updates)
            (api/column-will-change? :embedding_params dash-before-update dash-updates))
    (validation/check-embedding-enabled)
    (api/check-superuser)))

(api/defendpoint DELETE 
  "Delete a Dashboard.
  This will remove also any questions/models/segments/metrics that use this database."
  [id]
  {id ms/PositiveInt}
  (log/warn (str "DELETE /api/dashboard/:id is deprecated. Instead of deleting a Dashboard, you should change its "
                 "`archived` value via PUT /api/dashboard/:id."))
  (let [dashboard (api/write-check :model/Dashboard id)]
    (t2/delete! :model/Dashboard :id id)
    (events/publish-event! :event/dashboard-delete {:object dashboard :user-id api/*current-user-id*}))
  api/generic-204-no-content)

(defn- param-target->field-id [target query]
  (when-let [field-clause (params/param-target->field-clause target {:dataset_query query})]
    (mbql.u/match-one field-clause [:field (id :guard integer?) _] id)))

(mu/defn ^:private check-parameter-mapping-permissions
  {:added "0.41.0"}
  [parameter-mappings :- [:sequential dashboard-card/ParamMapping]]
  (when (seq parameter-mappings)
    ;; calculate a set of all Field IDs referenced by parameter mappings; then from those Field IDs calculate a set of
    ;; all Table IDs to which those Fields belong. This is done in a batched fashion so we can avoid N+1 query issues
    ;; if there happen to be a lot of parameters
    (let [card-ids              (into #{}
                                      (comp (map :card-id)
                                            (remove nil?))
                                      parameter-mappings)]
      (when (seq card-ids)
        (let [card-id->query        (t2/select-pk->fn :dataset_query Card :id [:in card-ids])
              field-ids             (set (for [{:keys [target card-id]} parameter-mappings
                                               :when                    card-id
                                               :let                     [query    (or (card-id->query card-id)
                                                                                      (throw (ex-info (tru "Card {0} does not exist or does not have a valid query."
                                                                                                           card-id)
                                                                                                      {:status-code 404
                                                                                                       :card-id     card-id})))
                                                                         field-id (param-target->field-id target query)]
                                               :when                    field-id]
                                           field-id))
              table-ids             (when (seq field-ids)
                                      (t2/select-fn-set :table_id Field :id [:in field-ids]))
              table-id->database-id (when (seq table-ids)
                                      (t2/select-pk->fn :db_id Table :id [:in table-ids]))]
          (doseq [table-id table-ids
                  :let     [database-id (table-id->database-id table-id)]]
            ;; check whether we'd actually be able to query this Table (do we have ad-hoc data perms for it?)
            (when-not (query-perms/can-query-table? database-id table-id)
              (throw (ex-info (tru "You must have data permissions to add a parameter referencing the Table {0}."
                                   (pr-str (t2/select-one-fn :name Table :id table-id)))
                              {:status-code        403
                               :database-id        database-id
                               :table-id           table-id
                               :actual-permissions @api/*current-user-permissions-set*})))))))))

(defn- existing-parameter-mappings
  [dashboard-id]
  (m/map-vals (fn [mappings]
                (into #{} (map #(select-keys % [:target :parameter_id])) mappings))
              (t2/select-pk->fn :parameter_mappings DashboardCard :dashboard_id dashboard-id)))

(defn- check-updated-parameter-mapping-permissions
  [dashboard-id dashcards]
  (let [dashcard-id->existing-mappings (existing-parameter-mappings dashboard-id)
        existing-mapping?              (fn [dashcard-id mapping]
                                         (let [[mapping]         (mi/normalize-parameters-list [mapping])
                                               existing-mappings (get dashcard-id->existing-mappings dashcard-id)]
                                           (contains? existing-mappings (select-keys mapping [:target :parameter_id]))))
        new-mappings                   (for [{mappings :parameter_mappings, dashcard-id :id} dashcards
                                             mapping mappings
                                             :when (not (existing-mapping? dashcard-id mapping))]
                                         (assoc mapping :dashcard-id dashcard-id))
        ;; need to add the appropriate `:card-id` for all the new mappings we're going to check.
        dashcard-id->card-id           (when (seq new-mappings)
                                         (t2/select-pk->fn :card_id DashboardCard
                                           :dashboard_id dashboard-id
                                           :id           [:in (set (map :dashcard-id new-mappings))]))
        new-mappings                   (for [{:keys [dashcard-id], :as mapping} new-mappings]
                                         (assoc mapping :card-id (get dashcard-id->card-id dashcard-id)))]
    (check-parameter-mapping-permissions new-mappings)))

(defn- create-dashcards!
  [dashboard dashcards]
  (doseq [{:keys [card_id]} dashcards
          :when  (pos-int? card_id)]
    (api/check-not-archived (api/read-check Card card_id)))
  (check-parameter-mapping-permissions (for [{:keys [card_id parameter_mappings]} dashcards
                                             mapping parameter_mappings]
                                        (assoc mapping :card-id card_id)))
  (api/check-500 (dashboard/add-dashcards! dashboard dashcards)))

(defn- update-dashcards! [dashboard dashcards]
  (check-updated-parameter-mapping-permissions (:id dashboard) dashcards)
  ;; transform the dashcard data to the format of the DashboardCard model
  ;; so update-dashcards! can compare them with existing dashcards
  (dashboard/update-dashcards! dashboard (map dashboard-card/from-parsed-json dashcards))
  dashcards)

(defn- delete-dashcards! [dashcard-ids]
  (let [dashboard-cards (t2/select DashboardCard :id [:in dashcard-ids])]
    (dashboard-card/delete-dashboard-cards! dashcard-ids)
    dashboard-cards))

(defn- do-update-dashcards!
  [dashboard current-cards new-cards]
  (let [{:keys [to-create to-update to-delete]} (u/classify-changes current-cards new-cards)]
    (when (seq to-update)
      (update-dashcards! dashboard to-update))
    {:deleted-dashcards (when (seq to-delete)
                          (delete-dashcards! (map :id to-delete)))
     :created-dashcards (when (seq to-create)
                          (create-dashcards! dashboard to-create))}))

(def ^:private UpdatedDashboardCard
  [:map
   ;; id can be negative, it indicates a new card and BE should create them
   [:id                                  int?]
   [:size_x                              ms/PositiveInt]
   [:size_y                              ms/PositiveInt]
   [:row                                 ms/IntGreaterThanOrEqualToZero]
   [:col                                 ms/IntGreaterThanOrEqualToZero]
   [:parameter_mappings {:optional true} [:maybe [:sequential [:map
                                                               [:parameter_id ms/NonBlankString]
                                                               [:target       :any]]]]]
   [:series             {:optional true} [:maybe [:sequential map?]]]])

(def ^:private UpdatedDashboardTab
  [:map
   ;; id can be negative, it indicates a new card and BE should create them
   [:id   ms/Int]
   [:name ms/NonBlankString]])

(defn- track-dashcard-and-tab-events!
  [{dashboard-id :id :as dashboard}
   {:keys [created-dashcards deleted-dashcards
           created-tab-ids deleted-tab-ids total-num-tabs]}]
  ;; Dashcard events
  (when (seq deleted-dashcards)
    (events/publish-event! :event/dashboard-remove-cards
                           {:object dashboard :user-id api/*current-user-id* :dashcards deleted-dashcards}))
  (when (seq created-dashcards)
    (events/publish-event! :event/dashboard-add-cards
                           {:object dashboard :user-id api/*current-user-id* :dashcards created-dashcards})
    (for [{:keys [card_id]} created-dashcards
          :when             (pos-int? card_id)]
      (snowplow/track-event! ::snowplow/question-added-to-dashboard
                             api/*current-user-id*
                             {:dashboard-id dashboard-id :question-id card_id :user-id api/*current-user-id*})))
  ;; Tabs events
  (when (seq deleted-tab-ids)
    (snowplow/track-event! ::snowplow/dashboard-tab-deleted
                           api/*current-user-id*
                           {:dashboard-id   dashboard-id
                            :num-tabs       (count deleted-tab-ids)
                            :total-num-tabs total-num-tabs}))
  (when (seq created-tab-ids)
    (snowplow/track-event! ::snowplow/dashboard-tab-created
                           api/*current-user-id*
                           {:dashboard-id   dashboard-id
                            :num-tabs       (count created-tab-ids)
                            :total-num-tabs total-num-tabs})))

(defn- update-dashboard
  [id {:keys [dashcards tabs] :as dash-updates}]
  (span/with-span!
    {:name       "update-dashboard"
     :attributes {:dashboard/id id}}
    (let [current-dash               (api/write-check Dashboard id)
          changes-stats              (atom nil)
          ;; tabs are always sent in production as well when dashcards are updated, but there are lots of
          ;; tests that exclude it. so this only checks for dashcards
          update-dashcards-and-tabs? (contains? dash-updates :dashcards)]
      (collection/check-allowed-to-change-collection current-dash dash-updates)
      (check-allowed-to-change-embedding current-dash dash-updates)
      (api/check-500
        (do
          (t2/with-transaction [_conn]
            ;; If the dashboard has an updated position, or if the dashboard is moving to a new collection, we might need to
            ;; adjust the collection position of other dashboards in the collection
            (api/maybe-reconcile-collection-position! current-dash dash-updates)
            (when-let [updates (not-empty
                                 (u/select-keys-when
                                   dash-updates
                                   :present #{:description :position :collection_id :collection_position :cache_ttl}
                                   :non-nil #{:name :parameters :caveats :points_of_interest :show_in_getting_started :enable_embedding
                                              :embedding_params :archived :auto_apply_filters}))]
              (t2/update! Dashboard id updates))
            (when update-dashcards-and-tabs?
              (when (not (false? (:archived false)))
                (api/check-not-archived current-dash))
              (let [{current-dashcards :dashcards
                     current-tabs      :tabs
                     :as               hydrated-current-dash} (t2/hydrate current-dash [:dashcards :series :card] :tabs)
                    _                       (when (and (seq current-tabs)
                                                       (not (every? #(some? (:dashboard_tab_id %)) dashcards)))
                                              (throw (ex-info (tru "This dashboard has tab, makes sure every card has a tab")
                                                              {:status-code 400})))
                    new-tabs                (map-indexed (fn [idx tab] (assoc tab :position idx)) tabs)
                    {:keys [old->new-tab-id
                            deleted-tab-ids]
                     :as   tabs-changes-stats} (dashboard-tab/do-update-tabs! (:id current-dash) current-tabs new-tabs)
                    deleted-tab-ids         (set deleted-tab-ids)
                    current-dashcards       (remove (fn [dashcard]
                                                      (contains? deleted-tab-ids (:dashboard_tab_id dashcard)))
                                                    current-dashcards)
                    new-dashcards           (cond->> dashcards
                                                     ;; fixup the temporary tab ids with the real ones
                                                     (seq old->new-tab-id)
                                                     (map (fn [card]
                                                            (if-let [real-tab-id (get old->new-tab-id (:dashboard_tab_id card))]
                                                              (assoc card :dashboard_tab_id real-tab-id)
                                                              card))))
                    dashcards-changes-stats (do-update-dashcards! hydrated-current-dash current-dashcards new-dashcards)]
                (reset! changes-stats
                        (merge
                          (select-keys tabs-changes-stats [:created-tab-ids :deleted-tab-ids :total-num-tabs])
                          (select-keys dashcards-changes-stats [:created-dashcards :deleted-dashcards]))))))
          true))
      (let [dashboard (t2/select-one :model/Dashboard id)]
        ;; skip publishing the event if it's just a change in its collection position
        (when-not (= #{:collection_position}
                     (set (keys dash-updates)))
          (events/publish-event! :event/dashboard-update {:object dashboard :user-id api/*current-user-id*}))
        (track-dashcard-and-tab-events! dashboard @changes-stats)
        (-> dashboard
            hydrate-dashboard-details
            (assoc :last-edit-info (last-edit/edit-information-for-user @api/*current-user*)))))))

(api/defendpoint PUT 
  "Update a Dashboard, and optionally the `dashcards` and `tabs` of a Dashboard. The request body should be a JSON object with the same
  structure as the response from `GET /api/dashboard/:id`."
  [id :as {{:keys [description name parameters caveats points_of_interest show_in_getting_started enable_embedding
                   embedding_params position archived collection_id collection_position cache_ttl dashcards tabs]
            :as dash-updates} :body}]
  {id                      ms/PositiveInt
   name                    [:maybe ms/NonBlankString]
   description             [:maybe :string]
   caveats                 [:maybe :string]
   points_of_interest      [:maybe :string]
   show_in_getting_started [:maybe :boolean]
   enable_embedding        [:maybe :boolean]
   embedding_params        [:maybe ms/EmbeddingParams]
   parameters              [:maybe [:sequential ms/Parameter]]
   position                [:maybe ms/PositiveInt]
   archived                [:maybe :boolean]
   collection_id           [:maybe ms/PositiveInt]
   collection_position     [:maybe ms/PositiveInt]
   cache_ttl               [:maybe ms/PositiveInt]
   dashcards               [:maybe (ms/maps-with-unique-key [:sequential UpdatedDashboardCard] :id)]
   tabs                    [:maybe (ms/maps-with-unique-key [:sequential UpdatedDashboardTab] :id)]}
  (update-dashboard id dash-updates))

(api/defendpoint PUT 
  "(DEPRECATED -- Use the `PUT /api/dashboard/:id` endpoint instead.)
   Update `Cards` and `Tabs` on a Dashboard. Request body should have the form:
    {:cards        [{:id                 ... ; DashboardCard ID
                     :size_x             ...
                     :size_y             ...
                     :row                ...
                     :col                ...
                     :parameter_mappings ...
                     :series             [{:id 123
                                           ...}]}
                     ...]
     :tabs [{:id       ... ; DashboardTab ID
                     :name     ...}]}"
  [id :as {{:keys [cards tabs]} :body}]
  {id           ms/PositiveInt
   cards        (ms/maps-with-unique-key [:sequential UpdatedDashboardCard] :id)
   ;; tabs should be required in production, making it optional because lots of
   ;; e2e tests curerntly doesn't include it
   tabs [:maybe (ms/maps-with-unique-key [:sequential UpdatedDashboardTab] :id)]}
  (log/warn
   "DELETE /api/dashboard/:id/cards is deprecated. Use PUT /api/dashboard/:id instead.")
  (let [dashboard (update-dashboard id {:dashcards cards :tabs tabs})]
    {:cards (:dashcards dashboard)
     :tabs  (:tabs dashboard)}))

(api/defendpoint GET 
  "Fetch `Revisions` for Dashboard with ID."
  [id]
  {id ms/PositiveInt}
  (api/read-check :model/Dashboard id)
  (revision/revisions+details :model/Dashboard id))

(api/defendpoint POST 
  "Revert a Dashboard to a prior `Revision`."
  [id :as {{:keys [revision_id]} :body}]
  {id ms/PositiveInt
   revision_id ms/PositiveInt}
  (api/write-check :model/Dashboard id)
  (revision/revert!
   {:entity      :model/Dashboard
    :id          id
    :user-id     api/*current-user-id*
    :revision-id revision_id}))

(api/defendpoint POST 
  "Generate publicly-accessible links for this Dashboard. Returns UUID to be used in public links. (If this
  Dashboard has already been shared, it will return the existing public link rather than creating a new one.) Public
  sharing must be enabled."
  [dashboard-id]
  {dashboard-id ms/PositiveInt}
  (api/check-superuser)
  (validation/check-public-sharing-enabled)
  (api/check-not-archived (api/read-check :model/Dashboard dashboard-id))
  {:uuid (or (t2/select-one-fn :public_uuid :model/Dashboard :id dashboard-id)
             (u/prog1 (str (random-uuid))
               (t2/update! :model/Dashboard dashboard-id
                           {:public_uuid       <>
                            :made_public_by_id api/*current-user-id*})))})

(api/defendpoint DELETE 
  "Delete the publicly-accessible link to this Dashboard."
  [dashboard-id]
  {dashboard-id ms/PositiveInt}
  (validation/check-has-application-permission :setting)
  (validation/check-public-sharing-enabled)
  (api/check-exists? :model/Dashboard :id dashboard-id, :public_uuid [:not= nil], :archived false)
  (t2/update! :model/Dashboard dashboard-id
              {:public_uuid       nil
               :made_public_by_id nil})
  {:status 204, :body nil})

(api/defendpoint GET 
  "Fetch a list of Dashboards with public UUIDs. These dashboards are publicly-accessible *if* public sharing is
  enabled."
  []
  (validation/check-has-application-permission :setting)
  (validation/check-public-sharing-enabled)
  (t2/select [:model/Dashboard :name :id :public_uuid], :public_uuid [:not= nil], :archived false))

(api/defendpoint GET 
  "Fetch a list of Dashboards where `enable_embedding` is `true`. The dashboards can be embedded using the embedding
  endpoints and a signed JWT."
  []
  (validation/check-has-application-permission :setting)
  (validation/check-embedding-enabled)
  (t2/select [:model/Dashboard :name :id], :enable_embedding true, :archived false))

(api/defendpoint GET 
  "Return related entities."
  [id]
  {id ms/PositiveInt}
  (-> (t2/select-one :model/Dashboard :id id) api/read-check related/related))

(api/defendpoint POST 
  "Save a denormalized description of dashboard into collection with ID `:parent-collection-id`."
  [parent-collection-id :as {dashboard :body}]
  {parent-collection-id ms/PositiveInt}
  (collection/check-write-perms-for-collection parent-collection-id)
  (let [dashboard (dashboard/save-transient-dashboard! dashboard parent-collection-id)]
    (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})
    dashboard))

(api/defendpoint POST 
  "Save a denormalized description of dashboard."
  [:as {dashboard :body}]
  (let [parent-collection-id (if api/*is-superuser?*
                               (:id (populate/get-or-create-root-container-collection))
                               (t2/select-one-fn :id 'Collection
                                                 :personal_owner_id api/*current-user-id*))
        dashboard (dashboard/save-transient-dashboard! dashboard parent-collection-id)]
    (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})
    dashboard))

(def ^:const result-limit
  1000)

(defn- get-template-tag
  [dimension card]
  (when-let [[_ tag] (mbql.u/check-clause :template-tag dimension)]
    (get-in card [:dataset_query :native :template-tags (u/qualified-name tag)])))

(defn- param-type->op [type]
  (if (get-in mbql.s/parameter-types [type :operator])
    (keyword (name type))
    :=))

(mu/defn ^:private param->fields
  [{:keys [mappings] :as param} :- mbql.s/Parameter]
  (for [{:keys [target] {:keys [card]} :dashcard} mappings
        :let  [[_ dimension] (->> (mbql.normalize/normalize-tokens target :ignore-path)
                                  (mbql.u/check-clause :dimension))]
        :when dimension
        :let  [ttag      (get-template-tag dimension card)
               dimension (condp mbql.u/is-clause? dimension
                           :field        dimension
                           :expression   dimension
                           :template-tag (:dimension ttag)
                           (log/error "cannot handle this dimension" {:dimension dimension}))
               field-id  (or
                          ;; Get the field id from the field-clause if it contains it. This is the common case
                          ;; for mbql queries.
                          (mbql.u/match-one dimension [:field (id :guard integer?) _] id)
                          ;; Attempt to get the field clause from the model metadata corresponding to the field.
                          ;; This is the common case for native queries in which mappings from original columns
                          ;; have been performed using model metadata.
                          (:id (qp.util/field->field-info dimension (:result_metadata card))))]
        :when field-id]
    {:field-id field-id
     :op       (param-type->op (:type param))
     :options  (merge (:options ttag)
                      (:options param))}))

(mu/defn ^:private chain-filter-constraints :- chain-filter/Constraints
  [dashboard constraint-param-key->value]
  (vec (for [[param-key value] constraint-param-key->value
             :let              [param (get-in dashboard [:resolved-params param-key])]
             :when             param
             field             (param->fields param)]
         (assoc field :value value))))

(defn filter-values-from-field-refs
  [dashboard param-key]
  (let [dashboard       (t2/hydrate dashboard :resolved-params)
        param           (get-in dashboard [:resolved-params param-key])
        results         (for [{:keys [target] {:keys [card]} :dashcard} (:mappings param)
                              :let [[_ dimension] (->> (mbql.normalize/normalize-tokens target :ignore-path)
                                                       (mbql.u/check-clause :dimension))]
                              :when dimension]
                          (custom-values/values-from-card card dimension))]
    (when-some [values (seq (distinct (mapcat :values results)))]
      (let [has_more_values (boolean (some true? (map :has_more_values results)))]
        {:values          (cond->> values
                                   (seq values)
                                   (sort-by (case (count (first values))
                                              2 second
                                              1 first)))
         :has_more_values has_more_values}))))

(mu/defn chain-filter :- ms/FieldValuesResult
  "C H A I N filters!
  Used to query for values that populate chained filter dropdowns and text search boxes."
  ([dashboard param-key constraint-param-key->value]
   (chain-filter dashboard param-key constraint-param-key->value nil))
  ([dashboard                   :- ms/Map
    param-key                   :- ms/NonBlankString
    constraint-param-key->value :- ms/Map
    query                       :- [:maybe ms/NonBlankString]]
   (let [dashboard   (t2/hydrate dashboard :resolved-params)
         constraints (chain-filter-constraints dashboard constraint-param-key->value)
         param       (get-in dashboard [:resolved-params param-key])
         field-ids   (map :field-id (param->fields param))]
     (if (empty? field-ids)
       (or (filter-values-from-field-refs dashboard param-key)
           (throw (ex-info (tru "Parameter {0} does not have any Fields associated with it" (pr-str param-key))
                           {:param       (get (:resolved-params dashboard) param-key)
                            :status-code 400})))
       (try
         (let [results         (map (if (seq query)
                                      #(chain-filter/chain-filter-search % constraints query :limit result-limit)
                                      #(chain-filter/chain-filter % constraints :limit result-limit))
                                    field-ids)
               values          (distinct (mapcat :values results))
               has_more_values (boolean (some true? (map :has_more_values results)))]
           ;; results can come back as [[v] ...] *or* as [[orig remapped] ...]. Sort by remapped value if it's there
           {:values          (cond->> values
                                      (seq values)
                                      (sort-by (case (count (first values))
                                                 2 second
                                                 1 first)))
            :has_more_values has_more_values})
         (catch clojure.lang.ExceptionInfo e
           (if (= (:type (u/all-ex-data e)) qp.error-type/missing-required-permissions)
             (api/throw-403 e)
             (throw e))))))))

(mu/defn param-values
  ([dashboard param-key constraint-param-key->value]
   (param-values dashboard param-key constraint-param-key->value nil))
  ([dashboard                   :- :map
    param-key                   :- ms/NonBlankString
    constraint-param-key->value :- :map
    query                       :- [:maybe ms/NonBlankString]]
   (let [dashboard (t2/hydrate dashboard :resolved-params)
         param     (get (:resolved-params dashboard) param-key)]
     (when-not param
       (throw (ex-info (tru "Dashboard does not have a parameter with the ID {0}" (pr-str param-key))
                       {:resolved-params (keys (:resolved-params dashboard))
                        :status-code     400})))
     (custom-values/parameter->values
       param
       query
       (fn [] (chain-filter dashboard param-key constraint-param-key->value query))))))

(api/defendpoint GET 
  "Fetch possible values of the parameter whose ID is `:param-key`. If the values come directly from a query, optionally
  restrict these values by passing query parameters like `other-parameter=value` e.g.
    ;; fetch values for Dashboard 1 parameter 'abc' that are possible when parameter 'def' is set to 100
    GET /api/dashboard/1/params/abc/values?def=100"
  [id param-key :as {constraint-param-key->value :query-params}]
  {id ms/PositiveInt}
  (let [dashboard (api/read-check :model/Dashboard id)]
    ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.
    (binding [qp.perms/*param-values-query* true]
      (param-values dashboard param-key constraint-param-key->value))))

(api/defendpoint GET 
  "Fetch possible values of the parameter whose ID is `:param-key` that contain `:query`. Optionally restrict
  these values by passing query parameters like `other-parameter=value` e.g.
    ;; fetch values for Dashboard 1 parameter 'abc' that contain 'Cam' and are possible when parameter 'def' is set
    ;; to 100
     GET /api/dashboard/1/params/abc/search/Cam?def=100
  Currently limited to first 1000 results."
  [id param-key query :as {constraint-param-key->value :query-params}]
  {id    ms/PositiveInt
   query ms/NonBlankString}
  (let [dashboard (api/read-check :model/Dashboard id)]
    ;; If a user can read the dashboard, then they can lookup filters. This also works with sandboxing.
    (binding [qp.perms/*param-values-query* true]
      (param-values dashboard param-key constraint-param-key->value query))))

(api/defendpoint GET 
  "Utility endpoint for powering Dashboard UI. Given some set of `filtered` Field IDs (presumably Fields used in
  parameters) and a set of `filtering` Field IDs that will be used to restrict values of `filtered` Fields, for each
  `filtered` Field ID return the subset of `filtering` Field IDs that would actually be used in a chain filter query
  with these Fields.
  e.g. in a chain filter query like
  GET /api/dashboard/10/params/PARAM_1/values?PARAM_2=100
  Assume `PARAM_1` maps to Field 1 and `PARAM_2` maps to Fields 2 and 3. The underlying MBQL query may or may not
  filter against Fields 2 and 3, depending on whether an FK relationship that lets us create a join against Field 1
  can be found. You can use this endpoint to determine which of those Fields is actually used:
  GET /api/dashboard/params/valid-filter-fields?filtered=1&filtering=2&filtering=3
  ;; ->
  {1 [2 3]}
  Results are returned as a map of
  `filtered` Field ID -> subset of `filtering` Field IDs that would be used in chain filter query"
  [:as {{:keys [filtered filtering]} :params}]
  {filtered  [:or ms/IntGreaterThanOrEqualToZero
              [:+ ms/IntGreaterThanOrEqualToZero]]
   filtering [:maybe [:or ms/IntGreaterThanOrEqualToZero
                      [:+ ms/IntGreaterThanOrEqualToZero]]]}
  (let [filtered-field-ids  (if (sequential? filtered) (set filtered) #{filtered})
        filtering-field-ids (if (sequential? filtering) (set filtering) #{filtering})]
    (doseq [field-id (set/union filtered-field-ids filtering-field-ids)]
      (api/read-check Field field-id))
    (into {} (for [field-id filtered-field-ids]
               [field-id (sort (chain-filter/filterable-field-ids field-id filtering-field-ids))]))))

(def ParameterWithID
  (mu/with-api-error-message
    [:and
     [:map
      [:id ms/NonBlankString]]
     [:map-of :keyword :any]]
    (deferred-tru "value must be a parameter map with an 'id' key")))

(api/defendpoint GET 
  "Fetches the values for filling in execution parameters. Pass PK parameters and values to select."
  [dashboard-id dashcard-id parameters]
  {dashboard-id ms/PositiveInt
   dashcard-id  ms/PositiveInt
   parameters   ms/JSONString}
  (api/read-check :model/Dashboard dashboard-id)
  (actions.execution/fetch-values
   (api/check-404 (action/dashcard->action dashcard-id))
   (json/parse-string parameters)))

(api/defendpoint POST 
  "Execute the associated Action in the context of a `Dashboard` and `DashboardCard` that includes it.
   `parameters` should be the mapped dashboard parameters with values.
   `extra_parameters` should be the extra, user entered parameter values."
  [dashboard-id dashcard-id :as {{:keys [parameters], :as _body} :body}]
  {dashboard-id ms/PositiveInt
   dashcard-id  ms/PositiveInt
   parameters  [:maybe [:map-of :keyword :any]]}
  (api/read-check :model/Dashboard dashboard-id)
  ;; Undo middleware string->keyword coercion
  (actions.execution/execute-dashcard! dashboard-id dashcard-id (update-keys parameters name)))

(api/defendpoint POST 
  "Run the query associated with a Saved Question (`Card`) in the context of a `Dashboard` that includes it."
  [dashboard-id dashcard-id card-id :as {{:keys [parameters], :as body} :body}]
  {dashboard-id  ms/PositiveInt
   dashcard-id   ms/PositiveInt
   card-id       ms/PositiveInt
   parameters    [:maybe [:sequential ParameterWithID]]}
  (m/mapply qp.dashboard/run-query-for-dashcard-async
            (merge
             body
             {:dashboard-id dashboard-id
              :card-id      card-id
              :dashcard-id  dashcard-id})))

(api/defendpoint POST 
  "Run the query associated with a Saved Question (`Card`) in the context of a `Dashboard` that includes it, and return
  its results as a file in the specified format.
  `parameters` should be passed as query parameter encoded as a serialized JSON string (this is because this endpoint
  is normally used to power 'Download Results' buttons that use HTML `form` actions)."
  [dashboard-id dashcard-id card-id export-format :as {{:keys [parameters], :as request-parameters} :params}]
  {dashboard-id  ms/PositiveInt
   dashcard-id   ms/PositiveInt
   card-id       ms/PositiveInt
   parameters    [:maybe ms/JSONString]
   export-format api.dataset/ExportFormat}
  (m/mapply qp.dashboard/run-query-for-dashcard-async
            (merge
             request-parameters
             {:dashboard-id  dashboard-id
              :card-id       card-id
              :dashcard-id   dashcard-id
              :export-format export-format
              :parameters    (json/parse-string parameters keyword)
              :context       (api.dataset/export-format->context export-format)
              :constraints   nil
              ;; TODO -- passing this `:middleware` map is a little repetitive, need to think of a way to not have to
              ;; specify this all over the codebase any time we want to do a query with an export format. Maybe this
              ;; should be the default if `export-format` isn't `:api`?
              :middleware    {:process-viz-settings?  true
                              :skip-results-metadata? true
                              :ignore-cached-results? true
                              :format-rows?           false
                              :js-int-to-string?      false}})))

(api/defendpoint POST 
  "Run a pivot table query for a specific DashCard."
  [dashboard-id dashcard-id card-id :as {{:keys [parameters], :as body} :body}]
  {dashboard-id ms/PositiveInt
   dashcard-id  ms/PositiveInt
   card-id      ms/PositiveInt
   parameters   [:maybe [:sequential ParameterWithID]]}
  (m/mapply qp.dashboard/run-query-for-dashcard-async
            (merge
             body
             {:dashboard-id dashboard-id
              :card-id      card-id
              :dashcard-id  dashcard-id
              :qp-runner    qp.pivot/run-pivot-query})))

(api/define-routes)

(ns metabase.api.database
  (:require
   [clojure.string :as str]
   [compojure.core :refer [DELETE GET POST PUT]]
   [medley.core :as m]
   [metabase.analytics.snowplow :as snowplow]
   [metabase.api.common :as api]
   [metabase.api.table :as api.table]
   [metabase.config :as config]
   [metabase.db.connection :as mdb.connection]
   [metabase.db.query :as mdb.query]
   [metabase.driver :as driver]
   [metabase.driver.ddl.interface :as ddl.i]
   [metabase.driver.h2 :as h2]
   [metabase.driver.util :as driver.u]
   [metabase.events :as events]
   [metabase.lib.schema.id :as lib.schema.id]
   [metabase.mbql.util :as mbql.u]
   [metabase.models.card :refer [Card]]
   [metabase.models.collection :as collection :refer [Collection]]
   [metabase.models.database
    :as database
    :refer [Database protected-password]]
   [metabase.models.field :refer [Field readable-fields-only]]
   [metabase.models.interface :as mi]
   [metabase.models.permissions :as perms]
   [metabase.models.persisted-info :as persisted-info]
   [metabase.models.secret :as secret]
   [metabase.models.setting :as setting :refer [defsetting]]
   [metabase.models.table :refer [Table]]
   [metabase.plugins.classloader :as classloader]
   [metabase.public-settings :as public-settings]
   [metabase.public-settings.premium-features :as premium-features]
   [metabase.sample-data :as sample-data]
   [metabase.sync.analyze :as analyze]
   [metabase.sync.field-values :as field-values]
   [metabase.sync.schedules :as sync.schedules]
   [metabase.sync.sync-metadata :as sync-metadata]
   [metabase.sync.util :as sync-util]
   [metabase.task.persist-refresh :as task.persist-refresh]
   [metabase.upload :as upload]
   [metabase.util :as u]
   [metabase.util.cron :as u.cron]
   [metabase.util.honey-sql-2 :as h2x]
   [metabase.util.i18n :refer [deferred-tru trs tru]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(def DBEngineString
  (mu/with-api-error-message
   [:and
    ms/NonBlankString
    [:fn
     {:error/message "Valid database engine"}
     #(u/ignore-exceptions (driver/the-driver %))]]
   (deferred-tru "value must be a valid database engine.")))

(defn- add-tables [dbs]
  (let [db-id->tables (group-by :db_id (filter mi/can-read? (t2/select Table
                                                              :active          true
                                                              :db_id           [:in (map :id dbs)]
                                                              :visibility_type nil
                                                              {:order-by [[:%lower.schema :asc]
                                                                          [:%lower.display_name :asc]]})))]
    (for [db dbs]
      (assoc db :tables (get db-id->tables (:id db) [])))))

(mu/defn ^:private add-native-perms-info :- [:maybe
                                             [:sequential
                                              [:map
                                               [:native_permissions [:enum :write :none]]]]]
  "For each database in DBS add a `:native_permissions` field describing the current user's permissions for running
  native (e.g. SQL) queries. Will be either `:write` or `:none`. `:write` means you can run ad-hoc native queries,
  and save new Cards with native queries; `:none` means you can do neither.
  For the curious: the use of `:write` and `:none` is mainly for legacy purposes, when we had data-access-based
  permissions; there was a specific option where you could give a Perms Group permissions to run existing Cards with
  native queries, but not to create new ones. With the advent of what is currently being called 'Space-Age
  Permissions', all Cards' permissions are based on their parent Collection, removing the need for native read perms."
  [dbs :- [:maybe [:sequential :map]]]
  (for [db dbs]
    (assoc db :native_permissions (if (perms/set-has-full-permissions? @api/*current-user-permissions-set*
                                        (perms/adhoc-native-query-path (u/the-id db)))
                                    :write
                                    :none))))

(defn- card-database-supports-nested-queries? [{{database-id :database, :as database} :dataset_query, :as _card}]
  (when database-id
    (when-let [driver (driver.u/database->driver database-id)]
      (driver/database-supports? driver :nested-queries database))))

(defn- card-has-ambiguous-columns?
  [{result-metadata :result_metadata, dataset-query :dataset_query}]
  (and (= (:type dataset-query) :native)
       (some (partial re-find #"_2$")
             (map (comp name :name) result-metadata))))

(defn- card-uses-unnestable-aggregation?
  [{{{aggregations :aggregation} :query} :dataset_query}]
  (mbql.u/match aggregations #{:cum-count :cum-sum}))

(defn card-can-be-used-as-source-query?
  [card]
  (and (card-database-supports-nested-queries? card)
       (not (or (card-uses-unnestable-aggregation? card)
                (card-has-ambiguous-columns? card)))))

(defn- ids-of-dbs-that-support-source-queries []
  (set (filter (fn [db-id]
                 (try
                   (when-let [db (t2/select-one Database :id db-id)]
                     (driver/database-supports? (:engine db) :nested-queries db))
                   (catch Throwable e
                     (log/error e (tru "Error determining whether Database supports nested queries")))))
               (t2/select-pks-set Database))))

(defn- source-query-cards
  [question-type & {:keys [additional-constraints xform], :or {xform identity}}]
  {:pre [(#{:card :dataset} question-type)]}
  (when-let [ids-of-dbs-that-support-source-queries (not-empty (ids-of-dbs-that-support-source-queries))]
    (transduce
     (comp (map (partial mi/do-after-select Card))
           (filter card-can-be-used-as-source-query?)
           xform)
     (completing conj #(t2/hydrate % :collection))
     []
     (mdb.query/reducible-query {:select   [:name :description :database_id :dataset_query :id :collection_id :result_metadata
                                            [{:select   [:status]
                                              :from     [:moderation_review]
                                              :where    [:and
                                                         [:= :moderated_item_type "card"]
                                                         [:= :moderated_item_id :report_card.id]
                                                         [:= :most_recent true]]
                                              :order-by [[:id :desc]]
                                              :limit    1}
                                             :moderated_status]]
                                 :from     [:report_card]
                                 :where    (into [:and
                                                  [:not= :result_metadata nil]
                                                  [:= :archived false]
                                                  [:= :dataset (= question-type :dataset)]
                                                  [:in :database_id ids-of-dbs-that-support-source-queries]
                                                  (collection/visible-collection-ids->honeysql-filter-clause
                                                   (collection/permissions-set->visible-collection-ids
                                                    @api/*current-user-permissions-set*))]
                                                 additional-constraints)
                                 :order-by [[:%lower.name :asc]]}))))

(defn- source-query-cards-exist?
  [question-type]
  (seq (source-query-cards question-type :xform (take 1))))

(defn- cards-virtual-tables
  [question-type & {:keys [include-fields?]}]
  (for [card (source-query-cards question-type)]
    (api.table/card->virtual-table card :include-fields? include-fields?)))

(defn- saved-cards-virtual-db-metadata [question-type & {:keys [include-tables? include-fields?]}]
  (when (public-settings/enable-nested-queries)
    (cond-> {:name               (trs "Saved Questions")
             :id                 lib.schema.id/saved-questions-virtual-database-id
             :features           #{:basic-aggregations}
             :is_saved_questions true}
      include-tables? (assoc :tables (cards-virtual-tables question-type
                                                           :include-fields? include-fields?)))))

(defn- add-saved-questions-virtual-database [dbs & options]
  (let [virtual-db-metadata (apply saved-cards-virtual-db-metadata :card options)]
    ;; only add the 'Saved Questions' DB if there are Cards that can be used
    (cond-> dbs
      (and (source-query-cards-exist? :card) virtual-db-metadata) (concat [virtual-db-metadata]))))

(defn- filter-databases-by-data-model-perms
  [dbs]
  (let [filtered-dbs
        (if-let [f (when config/ee-available?
                     (classloader/require 'metabase-enterprise.advanced-permissions.common)
                     (resolve 'metabase-enterprise.advanced-permissions.common/filter-databases-by-data-model-perms))]
          (f dbs)
          dbs)]
    (map
     (fn [db] (if (mi/can-read? db)
                db
                (select-keys db [:id :name :tables])))
     filtered-dbs)))

(defn- check-db-data-model-perms
  [db]
  (let [filtered-dbs (filter-databases-by-data-model-perms [db])]
    (api/check-403 (first filtered-dbs))))

(defn- uploadable-db?
  [db]
  (driver/database-supports? (driver.u/database->driver db) :uploads db))

(defn- add-can-upload-to-dbs
  [dbs]
  (let [uploads-db-id (public-settings/uploads-database-id)]
    (for [db dbs]
      (assoc db :can_upload (and (= (:id db) uploads-db-id)
                                 (upload/can-create-upload? db (public-settings/uploads-schema-name)))))))

(defn- dbs-list
  [& {:keys [include-tables?
             include-saved-questions-db?
             include-saved-questions-tables?
             include-editable-data-model?
             include-analytics?
             exclude-uneditable-details?
             include-only-uploadable?]}]
  (let [dbs (t2/select Database (merge {:order-by [:%lower.name :%lower.engine]}
                                       (when-not include-analytics?
                                         {:where [:= :is_audit false]})))
        filter-by-data-access? (not (or include-editable-data-model? exclude-uneditable-details?))]
    (cond-> (add-native-perms-info dbs)
      include-tables?              add-tables
      true                         add-can-upload-to-dbs
      include-editable-data-model? filter-databases-by-data-model-perms
      exclude-uneditable-details?  (#(filter mi/can-write? %))
      filter-by-data-access?       (#(filter mi/can-read? %))
      include-saved-questions-db?  (add-saved-questions-virtual-database :include-tables? include-saved-questions-tables?)
      ;; Perms checks for uploadable DBs are handled by exclude-uneditable-details? (see below)
      include-only-uploadable?     (#(filter uploadable-db? %)))))

(api/defendpoint GET 
  "Fetch all `Databases`.
  * `include=tables` means we should hydrate the Tables belonging to each DB. Default: `false`.
  * `saved` means we should include the saved questions virtual database. Default: `false`.
  * `include_editable_data_model` will only include DBs for which the current user has data model editing
    permissions. (If `include=tables`, this also applies to the list of tables in each DB). Should only be used if
    Enterprise Edition code is available the advanced-permissions feature is enabled.
  * `exclude_uneditable_details` will only include DBs for which the current user can edit the DB details. Has no
    effect unless Enterprise Edition code is available and the advanced-permissions feature is enabled.
  * `include_only_uploadable` will only include DBs into which Metabase can insert new data."
  [include saved include_editable_data_model exclude_uneditable_details include_only_uploadable include_analytics]
  {include                       (mu/with-api-error-message
                                   [:maybe [:= "tables"]]
                                   (deferred-tru "include must be either empty or the value 'tables'"))
   include_analytics             [:maybe :boolean]
   saved                         [:maybe :boolean]
   include_editable_data_model   [:maybe :boolean]
   exclude_uneditable_details    [:maybe :boolean]
   include_only_uploadable       [:maybe :boolean]}
  (let [include-tables?                 (= include "tables")
        include-saved-questions-tables? (and saved include-tables?)
        only-editable?                  (or include_only_uploadable exclude_uneditable_details)
        db-list-res                     (or (dbs-list :include-tables?                 include-tables?
                                                      :include-saved-questions-db?     saved
                                                      :include-saved-questions-tables? include-saved-questions-tables?
                                                      :include-editable-data-model?    include_editable_data_model
                                                      :exclude-uneditable-details?     only-editable?
                                                      :include-analytics?              include_analytics
                                                      :include-only-uploadable?        include_only_uploadable)
                                            [])]
   {:data  db-list-res
    :total (count db-list-res)}))

(mu/defn ^:private expanded-schedules [db :- (mi/InstanceOf Database)]
  {:cache_field_values (u.cron/cron-string->schedule-map (:cache_field_values_schedule db))
   :metadata_sync      (u.cron/cron-string->schedule-map (:metadata_sync_schedule db))})

(defn- add-expanded-schedules
  [db]
  (assoc db :schedules (expanded-schedules db)))

(defn- filter-sensitive-fields
  [fields]
  (remove #(= :sensitive (:visibility_type %)) fields))

(defn- get-database-hydrate-include
  [db include]
  (if-not include
    db
    (-> (t2/hydrate db (case include
                         "tables"        :tables
                         "tables.fields" [:tables [:fields [:target :has_field_values] :has_field_values]]))
        (update :tables (fn [tables]
                          (cond->> tables
                            ; filter hidden tables
                            true                        (filter (every-pred (complement :visibility_type) mi/can-read?))
                            ; filter hidden fields
                            (= include "tables.fields") (map #(update % :fields filter-sensitive-fields))))))))

(defn- add-can-upload
  [db]
  (assoc db :can_upload (and (= (u/the-id db) (public-settings/uploads-database-id))
                             (upload/can-create-upload? db (public-settings/uploads-schema-name)))))

(api/defendpoint GET 
  "Get a single Database with `id`. Optionally pass `?include=tables` or `?include=tables.fields` to include the Tables
  belonging to this database, or the Tables and Fields, respectively.  If the requestor has write permissions for the DB
  (i.e. is an admin or has data model permissions), then certain inferred secret values will also be included in the
  returned details (see [[metabase.models.secret/expand-db-details-inferred-secret-values]] for full details).
  Passing include_editable_data_model will only return tables for which the current user has data model editing
  permissions, if Enterprise Edition code is available and a token with the advanced-permissions feature is present.
  In addition, if the user has no data access for the DB (aka block permissions), it will return only the DB name, ID
  and tables, with no additional metadata."
  [id include include_editable_data_model exclude_uneditable_details]
  {id      ms/PositiveInt
   include [:maybe [:enum "tables" "tables.fields"]]}
  (let [include-editable-data-model? (Boolean/parseBoolean include_editable_data_model)
        exclude-uneditable-details?  (Boolean/parseBoolean exclude_uneditable_details)
        filter-by-data-access?       (not (or include-editable-data-model? exclude-uneditable-details?))
        database                     (api/check-404 (t2/select-one Database :id id))]
    (cond-> database
      filter-by-data-access?       api/read-check
      exclude-uneditable-details?  api/write-check
      true                         add-expanded-schedules
      true                         (get-database-hydrate-include include)
      true                         add-can-upload
      include-editable-data-model? check-db-data-model-perms
      (mi/can-write? database)     (->
                                    secret/expand-db-details-inferred-secret-values
                                    (assoc :can-manage true)))))

(def ^:private database-usage-models
  [:question :dataset :metric :segment])

(def ^:private always-false-hsql-expr
  [:= [:inline 1] [:inline 2]])

(defmulti ^:private database-usage-query
  {:arglists '([model database-id table-ids])}
  (fn [model _database-id _table-ids] (keyword model)))

(defmethod database-usage-query :question
  [_ db-id _table-ids]
  {:select [[:%count.* :question]]
   :from   [:report_card]
   :where  [:and
            [:= :database_id db-id]
            [:= :dataset false]]})

(defmethod database-usage-query :dataset
  [_ db-id _table-ids]
  {:select [[:%count.* :dataset]]
   :from   [:report_card]
   :where  [:and
            [:= :database_id db-id]
            [:= :dataset true]]})

(defmethod database-usage-query :metric
  [_ _db-id table-ids]
  {:select [[:%count.* :metric]]
   :from   [:metric]
   :where  (if table-ids
             [:in :table_id table-ids]
             always-false-hsql-expr)})

(defmethod database-usage-query :segment
  [_ _db-id table-ids]
  {:select [[:%count.* :segment]]
   :from   [:segment]
   :where  (if table-ids
             [:in :table_id table-ids]
             always-false-hsql-expr)})

(api/defendpoint GET 
  "Get usage info for a database.
  Returns a map with keys are models and values are the number of entities that use this database."
  [id]
  {id ms/PositiveInt}
  (api/check-superuser)
  (api/check-404 (t2/exists? Database :id id))
  (let [table-ids (t2/select-pks-set Table :db_id id)]
    (first (mdb.query/query
             {:select [:*]
              :from   (for [model database-usage-models
                            :let [query (database-usage-query model id table-ids)]
                            :when query]
                        [query model])}))))

(api/defendpoint GET ["/:virtual-db/metadata" :virtual-db (re-pattern (str lib.schema.id/saved-questions-virtual-database-id))]
  "Endpoint that provides metadata for the Saved Questions 'virtual' database. Used for fooling the frontend
   and allowing it to treat the Saved Questions virtual DB just like any other database."
  []
  (saved-cards-virtual-db-metadata :card :include-tables? true, :include-fields? true))

(defn- db-metadata [id include-hidden? include-editable-data-model? remove_inactive?]
  (let [db (-> (if include-editable-data-model?
                 (api/check-404 (t2/select-one Database :id id))
                 (api/read-check Database id))
               (t2/hydrate [:tables [:fields [:target :has_field_values] :has_field_values] :segments :metrics]))
        db (if include-editable-data-model?
             ;; We need to check data model perms after hydrating tables, since this will also filter out tables for
             ;; which the *current-user* does not have data model perms
             (check-db-data-model-perms db)
             db)]
    (-> db
        (update :tables (if include-hidden?
                          identity
                          (fn [tables]
                            (->> tables
                                 (remove :visibility_type)
                                 (map #(update % :fields filter-sensitive-fields))))))
        (update :tables (fn [tables]
                          (if-not include-editable-data-model?
                            ;; If we're filtering by data model perms, table perm checks were already done by
                            ;; check-db-data-model-perms
                            (filter mi/can-read? tables)
                            tables)))
        (update :tables (fn [tables]
                          (for [table tables]
                            (-> table
                                (update :segments (partial filter mi/can-read?))
                                (update :metrics  (partial filter mi/can-read?))))))
        (update :tables (if remove_inactive?
                          (fn [tables]
                            (filter :active tables))
                          identity)))))

(api/defendpoint GET 
  "Get metadata about a `Database`, including all of its `Tables` and `Fields`. Returns DB, fields, and field values.
  By default only non-hidden tables and fields are returned. Passing include_hidden=true includes them.
  Passing include_editable_data_model will only return tables for which the current user has data model editing
  permissions, if Enterprise Edition code is available and a token with the advanced-permissions feature is present.
  In addition, if the user has no data access for the DB (aka block permissions), it will return only the DB name, ID
  and tables, with no additional metadata."
  [id include_hidden include_editable_data_model remove_inactive]
  {id                          ms/PositiveInt
   include_hidden              [:maybe ms/BooleanString]
   include_editable_data_model [:maybe ms/BooleanString]
   remove_inactive             [:maybe ms/BooleanString]}
  (db-metadata id
               (Boolean/parseBoolean include_hidden)
               (Boolean/parseBoolean include_editable_data_model)
               (Boolean/parseBoolean remove_inactive)))

(defn- autocomplete-tables [db-id search-string limit]
  (t2/select [Table :id :db_id :schema :name]
    {:where    [:and [:= :db_id db-id]
                     [:= :active true]
                     [:like :%lower.name (u/lower-case-en search-string)]
                     [:= :visibility_type nil]]
     :order-by [[:%lower.name :asc]]
     :limit    limit}))

(defn- autocomplete-cards
  [database-id search-card-slug]
  (let [search-id   (re-find #"\d*" search-card-slug)
        search-name (-> (re-matches #"\d*-?(.*)" search-card-slug)
                        second
                        (str/replace #"-" " ")
                        u/lower-case-en)]
    (t2/select [Card :id :dataset :database_id :name :collection_id [:collection.name :collection_name]]
               {:where    [:and
                           [:= :report_card.database_id database-id]
                           [:= :report_card.archived false]
                           (cond
                             ;; e.g. search-string = "123"
                             (and (not-empty search-id) (empty? search-name))
                             [:like
                              (h2x/cast (if (= (mdb.connection/db-type) :mysql) :char :text) :report_card.id)
                              (str search-id "%")]
                             ;; e.g. search-string = "123-foo"
                             (and (not-empty search-id) (not-empty search-name))
                             [:and
                              [:= :report_card.id (Integer/parseInt search-id)]
                              ;; this is a prefix match to be consistent with substring matches on the entire slug
                              [:like [:lower :report_card.name] (str search-name "%")]]
                             ;; e.g. search-string = "foo"
                             (and (empty? search-id) (not-empty search-name))
                             [:like [:lower :report_card.name] (str "%" search-name "%")])]
                :left-join [[:collection :collection] [:= :collection.id :report_card.collection_id]]
                :order-by [[:dataset :desc]         ; prioritize models
                           [:report_card.id :desc]] ; then most recently created
                :limit    50})))

(defn- autocomplete-fields [db-id search-string limit]
  (t2/select [Field :name :base_type :semantic_type :id :table_id [:table.name :table_name]]
             :metabase_field.active          true
             :%lower.metabase_field/name     [:like (u/lower-case-en search-string)]
             :metabase_field.visibility_type [:not-in ["sensitive" "retired"]]
             :table.db_id                    db-id
             {:order-by  [[[:lower :metabase_field.name] :asc]
                          [[:lower :table.name] :asc]]
              :left-join [[:metabase_table :table] [:= :table.id :metabase_field.table_id]]
              :limit     limit}))

(defn- autocomplete-results [tables fields limit]
  (let [tbl-count   (count tables)
        fld-count   (count fields)
        take-tables (min tbl-count (- limit (/ fld-count 2)))
        take-fields (- limit take-tables)]
    (concat (for [{table-name :name} (take take-tables tables)]
              [table-name "Table"])
            (for [{:keys [table_name base_type semantic_type name]} (take take-fields fields)]
              [name (str table_name
                         " "
                         base_type
                         (when semantic_type
                           (str " " semantic_type)))]))))

(defn- autocomplete-suggestions
  [db-id match-string]
  (let [limit  50
        tables (filter mi/can-read? (autocomplete-tables db-id match-string limit))
        fields (readable-fields-only (autocomplete-fields db-id match-string limit))]
    (autocomplete-results tables fields limit)))

(def ^:private autocomplete-matching-options
  #{:substring :prefix :off})

(defsetting native-query-autocomplete-match-style
  (deferred-tru
    (str "Matching style for native query editor's autocomplete. Can be \"substring\", \"prefix\", or \"off\". "
         "Larger instances can have performance issues matching using substring, so can use prefix matching, "
         " or turn autocompletions off."))
  :visibility :public
  :type       :keyword
  :default    :substring
  :audit      :raw-value
  :setter     (fn [v]
                (let [v (cond-> v (string? v) keyword)]
                  (if (autocomplete-matching-options v)
                    (setting/set-value-of-type! :keyword :native-query-autocomplete-match-style v)
                    (throw (ex-info (tru "Invalid `native-query-autocomplete-match-style` option")
                                    {:option v
                                     :valid-options autocomplete-matching-options}))))))

(api/defendpoint GET 
  "Return a list of autocomplete suggestions for a given `prefix`, or `substring`. Should only specify one, but
  `substring` will have priority if both are present.
  This is intended for use with the ACE Editor when the User is typing raw SQL. Suggestions include matching `Tables`
  and `Fields` in this `Database`.
  Tables are returned in the format `[table_name \"Table\"]`;
  When Fields have a semantic_type, they are returned in the format `[field_name \"table_name base_type semantic_type\"]`
  When Fields lack a semantic_type, they are returned in the format `[field_name \"table_name base_type\"]`"
  [id prefix substring]
  {id        ms/PositiveInt
   prefix    [:maybe ms/NonBlankString]
   substring [:maybe ms/NonBlankString]}
  (api/read-check Database id)
  (when (and (str/blank? prefix) (str/blank? substring))
    (throw (ex-info (tru "Must include prefix or search") {:status-code 400})))
  (try
    (cond
      substring
      (autocomplete-suggestions id (str "%" substring "%"))
      prefix
      (autocomplete-suggestions id (str prefix "%")))
    (catch Throwable e
      (log/warn e (trs "Error with autocomplete: {0}" (ex-message e))))))

(api/defendpoint GET 
  "Return a list of `Card` autocomplete suggestions for a given `query` in a given `Database`.
  This is intended for use with the ACE Editor when the User is typing in a template tag for a `Card`, e.g. {{#...}}."
  [id query]
  {id    ms/PositiveInt
   query ms/NonBlankString}
  (api/read-check Database id)
  (try
    (->> (autocomplete-cards id query)
         (filter mi/can-read?)
         (map #(select-keys % [:id :name :dataset :collection_name])))
    (catch Throwable e
      (log/warn e (trs "Error with autocomplete: {0}" (ex-message e))))))

(api/defendpoint GET 
  "Get a list of all `Fields` in `Database`."
  [id]
  {id ms/PositiveInt}
  (api/read-check Database id)
  (let [fields (filter mi/can-read? (-> (t2/select [Field :id :name :display_name :table_id :base_type :semantic_type]
                                          :table_id        [:in (t2/select-fn-set :id Table, :db_id id)]
                                          :visibility_type [:not-in ["sensitive" "retired"]])
                                        (t2/hydrate :table)))]
    (for [{:keys [id name display_name table base_type semantic_type]} fields]
      {:id            id
       :name          name
       :display_name  display_name
       :base_type     base_type
       :semantic_type semantic_type
       :table_name    (:name table)
       :schema        (:schema table)})))

(api/defendpoint GET 
  "Get a list of all primary key `Fields` for `Database`."
  [id include_editable_data_model]
  {id ms/PositiveInt}
  (let [[db-perm-check field-perm-check] (if (Boolean/parseBoolean include_editable_data_model)
                                           [check-db-data-model-perms mi/can-write?]
                                           [api/read-check mi/can-read?])]
    (db-perm-check (t2/select-one Database :id id))
    (sort-by (comp u/lower-case-en :name :table)
             (filter field-perm-check (-> (database/pk-fields {:id id})
                                          (t2/hydrate :table))))))

(defn test-database-connection
  [engine {:keys [host port] :as details}, & {:keys [log-exception]
                                              :or   {log-exception true}}]
  {:pre [(some? engine)]}
  (let [engine  (keyword engine)
        details (assoc details :engine engine)]
    (try
      (cond
        (driver.u/can-connect-with-details? engine details :throw-exceptions)
        nil
        (and host port (u/host-port-up? host port))
        {:message (tru "Connection to ''{0}:{1}'' successful, but could not connect to DB."
                       host port)}
        (and host (u/host-up? host))
        {:message (tru "Connection to host ''{0}'' successful, but port {1} is invalid."
                       host port)
         :errors  {:port (deferred-tru "check your port settings")}}
        host
        {:message (tru "Host ''{0}'' is not reachable" host)
         :errors  {:host (deferred-tru "check your host settings")}}
        :else
        {:message (tru "Unable to connect to database.")})
      (catch Throwable e
        (when (and log-exception (not (some->> e ex-cause ex-data ::driver/can-connect-message?)))
          (log/error e (trs "Cannot connect to Database")))
        (if (-> e ex-data :message)
          (ex-data e)
          {:message (.getMessage e)})))))

(defn- supports-ssl?
  [driver]
  {:pre [(driver/available? driver)]}
  (let [driver-props (set (for [field (driver/connection-properties driver)]
                            (:name field)))]
    (contains? driver-props "ssl")))

(mu/defn ^:private test-connection-details :- :map
  "Try a making a connection to database `engine` with `details`.
  If the `details` has SSL explicitly enabled, go with that and do not accept plaintext connections. If it is disabled,
  try twice: once with SSL, and a second time without if the first fails. If either attempt is successful, returns
  the details used to successfully connect. Otherwise returns a map with the connection error message. (This map will
  also contain the key `:valid` = `false`, which you can use to distinguish an error from valid details.)"
  [engine  :- DBEngineString
   details :- :map]
  (let [;; Try SSL first if SSL is supported and not already enabled
        ;; If not successful or not applicable, details-with-ssl will be nil
        details-with-ssl (assoc details :ssl true)
        details-with-ssl (when (and (supports-ssl? (keyword engine))
                                    (not (true? (:ssl details)))
                                    (nil? (test-database-connection engine details-with-ssl :log-exception false)))
                           details-with-ssl)]
    (or
      ;; Opportunistic SSL
      details-with-ssl
      ;; Try with original parameters
      (some-> (test-database-connection engine details)
              (assoc :valid false))
      details)))

(api/defendpoint POST 
  "Add a new `Database`."
  [:as {{:keys [name engine details is_full_sync is_on_demand schedules auto_run_queries cache_ttl]} :body}]
  {name             ms/NonBlankString
   engine           DBEngineString
   details          ms/Map
   is_full_sync     [:maybe :boolean]
   is_on_demand     [:maybe :boolean]
   schedules        [:maybe sync.schedules/ExpandedSchedulesMap]
   auto_run_queries [:maybe :boolean]
   cache_ttl        [:maybe ms/PositiveInt]}
  (api/check-superuser)
  (when cache_ttl
    (api/check (premium-features/enable-cache-granular-controls?)
               [402 (tru (str "The cache TTL database setting is only enabled if you have a premium token with the "
                              "cache granular controls feature."))]))
  (let [is-full-sync?    (or (nil? is_full_sync)
                             (boolean is_full_sync))
        details-or-error (test-connection-details engine details)
        valid?           (not= (:valid details-or-error) false)]
    (if valid?
      ;; no error, proceed with creation. If record is inserted successfuly, publish a `:database-create` event.
      ;; Throw a 500 if nothing is inserted
      (u/prog1 (api/check-500 (first (t2/insert-returning-instances!
                                      Database
                                      (merge
                                       {:name         name
                                        :engine       engine
                                        :details      details-or-error
                                        :is_full_sync is-full-sync?
                                        :is_on_demand (boolean is_on_demand)
                                        :cache_ttl    cache_ttl
                                        :creator_id   api/*current-user-id*}
                                       (sync.schedules/schedule-map->cron-strings
                                        (if (:let-user-control-scheduling details)
                                          (sync.schedules/scheduling schedules)
                                          (sync.schedules/default-randomized-schedule)))
                                       (when (some? auto_run_queries)
                                         {:auto_run_queries auto_run_queries})))))
        (events/publish-event! :event/database-create {:object <> :user-id api/*current-user-id*})
        (snowplow/track-event! ::snowplow/database-connection-successful
                               api/*current-user-id*
                               {:database     engine
                                :database-id  (u/the-id <>)
                                :source       :admin
                                :dbms-version (:version (driver/dbms-version (keyword engine) <>))}))
      ;; failed to connect, return error
      (do
        (snowplow/track-event! ::snowplow/database-connection-failed
                               api/*current-user-id*
                               {:database engine :source :setup})
        {:status 400
         :body   (dissoc details-or-error :valid)}))))

(api/defendpoint POST 
  "Validate that we can connect to a database given a set of details."
  ;; TODO - why do we pass the DB in under the key `details`?
  [:as {{{:keys [engine details]} :details} :body}]
  {engine  DBEngineString
   details :map}
  (api/check-superuser)
  (let [details-or-error (test-connection-details engine details)]
    {:valid (not (false? (:valid details-or-error)))}))

(api/defendpoint POST 
  "Add the sample database as a new `Database`."
  []
  (api/check-superuser)
  (sample-data/add-sample-database!)
  (t2/select-one Database :is_sample true))

(defn- upsert-sensitive-fields
  [database details]
  (when details
    (merge (:details database)
           (reduce
            (fn [details k]
              (if (= protected-password (get details k))
                (m/update-existing details k (constantly (get-in database [:details k])))
                details))
            details
            (database/sensitive-fields-for-db database)))))

(api/defendpoint POST 
  "Attempt to enable model persistence for a database. If already enabled returns a generic 204."
  [id]
  {id ms/PositiveInt}
  (api/check (public-settings/persisted-models-enabled)
             400
             (tru "Persisting models is not enabled."))
  (api/let-404 [database (t2/select-one Database :id id)]
    (api/write-check database)
    (if (-> database :settings :persist-models-enabled)
      ;; todo: some other response if already persisted?
      api/generic-204-no-content
      (let [[success? error] (ddl.i/check-can-persist database)
            schema           (ddl.i/schema-name database (public-settings/site-uuid))]
        (if success?
          ;; do secrets require special handling to not clobber them or mess up encryption?
          (do (t2/update! Database id {:settings (assoc (:settings database) :persist-models-enabled true)})
              (task.persist-refresh/schedule-persistence-for-database!
                database
                (public-settings/persisted-model-refresh-cron-schedule))
              api/generic-204-no-content)
          (throw (ex-info (ddl.i/error->message error schema)
                          {:error error
                           :database (:name database)})))))))

(api/defendpoint POST 
  "Attempt to disable model persistence for a database. If already not enabled, just returns a generic 204."
  [id]
  {id ms/PositiveInt}
  (api/let-404 [database (t2/select-one Database :id id)]
    (api/write-check database)
    (if (-> database :settings :persist-models-enabled)
      (do (t2/update! Database id {:settings (dissoc (:settings database) :persist-models-enabled)})
          (persisted-info/mark-for-pruning! {:database_id id})
          (task.persist-refresh/unschedule-persistence-for-database! database)
          api/generic-204-no-content)
      ;; todo: a response saying this was a no-op? an error? same on the post to persist
      api/generic-204-no-content)))

(api/defendpoint PUT 
  "Update a `Database`."
  [id :as {{:keys [name engine details is_full_sync is_on_demand description caveats points_of_interest schedules
                   auto_run_queries refingerprint cache_ttl settings]} :body}]
  {id                 ms/PositiveInt
   name               [:maybe ms/NonBlankString]
   engine             [:maybe DBEngineString]
   refingerprint      [:maybe :boolean]
   details            [:maybe ms/Map]
   schedules          [:maybe sync.schedules/ExpandedSchedulesMap]
   description        [:maybe :string]   ; :string instead of ms/NonBlankString because we don't care
   caveats            [:maybe :string]   ; whether someone sets these to blank strings
   points_of_interest [:maybe :string]
   auto_run_queries   [:maybe :boolean]
   cache_ttl          [:maybe ms/PositiveInt]
   settings           [:maybe ms/Map]}
  ;; TODO - ensure that custom schedules and let-user-control-scheduling go in lockstep
  (let [existing-database (api/write-check (t2/select-one Database :id id))
        details           (some->> details
                                   (driver.u/db-details-client->server (or engine (:engine existing-database)))
                                   (upsert-sensitive-fields existing-database))
        ;; verify that we can connect to the database if `:details` OR `:engine` have changed.
        details-changed?  (some-> details (not= (:details existing-database)))
        engine-changed?   (some-> engine keyword (not= (:engine existing-database)))
        conn-error        (when (or details-changed? engine-changed?)
                            (test-database-connection (or engine (:engine existing-database))
                                                      (or details (:details existing-database))))
        full-sync?        (some-> is_full_sync boolean)]
    (if conn-error
      ;; failed to connect, return error
      {:status 400
       :body   conn-error}
      ;; no error, proceed with update
      (do
        ;; TODO - is there really a reason to let someone change the engine on an existing database?
        ;;       that seems like the kind of thing that will almost never work in any practical way
        ;; TODO - this means one cannot unset the description. Does that matter?
        (t2/update! Database id
                    (m/remove-vals
                      nil?
                      (merge
                        {:name               name
                         :engine             engine
                         :details            details
                         :refingerprint      refingerprint
                         :is_full_sync       full-sync?
                         :is_on_demand       (boolean is_on_demand)
                         :description        description
                         :caveats            caveats
                         :points_of_interest points_of_interest
                         :auto_run_queries   auto_run_queries}
                        ;; upsert settings with a PATCH-style update. `nil` key means unset the Setting.
                        (when (seq settings)
                          {:settings (into {}
                                           (remove (fn [[_k v]] (nil? v)))
                                           (merge (:settings existing-database) settings))})
                        (cond
                          ;; transition back to metabase managed schedules. the schedule
                          ;; details, even if provided, are ignored. database is the
                          ;; current stored value and check against the incoming details
                          (and (get-in existing-database [:details :let-user-control-scheduling])
                               (not (:let-user-control-scheduling details)))
                          (sync.schedules/schedule-map->cron-strings (sync.schedules/default-randomized-schedule))
                          ;; if user is controlling schedules
                          (:let-user-control-scheduling details)
                          (sync.schedules/schedule-map->cron-strings (sync.schedules/scheduling schedules))))))
        ;; do nothing in the case that user is not in control of
        ;; scheduling. leave them as they are in the db
        ;; unlike the other fields, folks might want to nil out cache_ttl. it should also only be settable on EE
        ;; with the advanced-config feature enabled.
        (when (premium-features/enable-cache-granular-controls?)
          (t2/update! Database id {:cache_ttl cache_ttl}))
        (let [db (t2/select-one Database :id id)]
          (events/publish-event! :event/database-update {:object db
                                                         :user-id api/*current-user-id*
                                                         :previous-object existing-database})
          ;; return the DB with the expanded schedules back in place
          (add-expanded-schedules db))))))

(api/defendpoint DELETE 
  "Delete a `Database`."
  [id]
  {id ms/PositiveInt}
  (api/check-superuser)
  (api/let-404 [db (t2/select-one Database :id id)]
    (api/check-403 (mi/can-write? db))
    (t2/delete! Database :id id)
    (events/publish-event! :event/database-delete {:object db :user-id api/*current-user-id*}))
  api/generic-204-no-content)

(api/defendpoint POST 
  "Trigger a manual update of the schema metadata for this `Database`."
  [id]
  {id ms/PositiveInt}
  ;; just wrap this in a future so it happens async
  (let [db (api/write-check (t2/select-one Database :id id))]
    (events/publish-event! :event/database-manual-sync {:object db :user-id api/*current-user-id*})
    (if-let [ex (try
                  ;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the
                  ;; purposes of creating a new H2 database.
                  (binding [h2/*allow-testing-h2-connections* true]
                    (driver.u/can-connect-with-details? (:engine db) (:details db) :throw-exceptions))
                  nil
                  (catch Throwable e
                    e))]
      (throw (ex-info (ex-message ex) {:status-code 422}))
      (do
        (future
          (sync-metadata/sync-db-metadata! db)
          (analyze/analyze-db! db))
        {:status :ok}))))

(api/defendpoint POST 
  "Manually set the initial sync status of the `Database` and corresponding
  tables to be `complete` (see #20863)"
  [id]
  {id ms/PositiveInt}
  ;; manual full sync needs to be async, but this is a simple update of `Database`
  (let [db     (api/write-check (t2/select-one Database :id id))
        tables (map api/write-check (:tables (first (add-tables [db]))))]
    (sync-util/set-initial-database-sync-complete! db)
    ;; avoid n+1
    (when-let [table-ids (seq (map :id tables))]
      (t2/update! Table {:id [:in table-ids]} {:initial_sync_status "complete"})))
  {:status :ok})

(def ^:dynamic *rescan-values-async*
  true)

(api/defendpoint POST 
  "Trigger a manual scan of the field values for this `Database`."
  [id]
  {id ms/PositiveInt}
  ;; just wrap this is a future so it happens async
  (let [db (api/write-check (t2/select-one Database :id id))]
    (events/publish-event! :event/database-manual-scan {:object db :user-id api/*current-user-id*})
    ;; Override *current-user-permissions-set* so that permission checks pass during sync. If a user has DB detail perms
    ;; but no data perms, they should stll be able to trigger a sync of field values. This is fine because we don't
    ;; return any actual field values from this API. (#21764)
    (binding [api/*current-user-permissions-set* (atom #{"/"})]
      (if *rescan-values-async*
        (future (field-values/update-field-values! db))
        (field-values/update-field-values! db))))
  {:status :ok})

(defn- database->field-values-ids [database-or-id]
  (map :id (mdb.query/query {:select    [[:fv.id :id]]
                             :from      [[:metabase_fieldvalues :fv]]
                             :left-join [[:metabase_field :f] [:= :fv.field_id :f.id]
                                         [:metabase_table :t] [:= :f.table_id :t.id]]
                             :where     [:= :t.db_id (u/the-id database-or-id)]})))

(defn- delete-all-field-values-for-database! [database-or-id]
  (when-let [field-values-ids (seq (database->field-values-ids database-or-id))]
    (t2/query-one {:delete-from :metabase_fieldvalues
                   :where       [:in :id field-values-ids]})))

(api/defendpoint POST 
  "Discards all saved field values for this `Database`."
  [id]
  {id ms/PositiveInt}
  (let [db (api/write-check (t2/select-one Database :id id))]
    (events/publish-event! :event/database-discard-field-values {:object db :user-id api/*current-user-id*})
    (delete-all-field-values-for-database! db))
  {:status :ok})

(defn- can-read-schema?
  [database-id schema-name]
  (or
   (perms/set-has-partial-permissions? @api/*current-user-permissions-set*
                                       (perms/data-perms-path database-id schema-name))
   (perms/set-has-full-permissions? @api/*current-user-permissions-set*
                                    (perms/data-model-write-perms-path database-id schema-name))))

(api/defendpoint GET 
  "Returns a list of all syncable schemas found for the database `id`."
  [id]
  {id ms/PositiveInt}
  (let [db (api/check-404 (t2/select-one Database id))]
    (api/check-403 (mi/can-write? db))
    (->> db
         (driver/syncable-schemas (:engine db))
         (vec)
         (sort))))

(api/defendpoint GET 
  "Returns a list of all the schemas with tables found for the database `id`. Excludes schemas with no tables."
  [id include_editable_data_model include_hidden]
  {id                          ms/PositiveInt
   include_editable_data_model [:maybe ms/BooleanValue]
   include_hidden              [:maybe ms/BooleanValue]}
  (let [filter-schemas (fn [schemas]
                         (if include_editable_data_model
                           (if-let [f (u/ignore-exceptions
                                       (classloader/require 'metabase-enterprise.advanced-permissions.common)
                                       (resolve 'metabase-enterprise.advanced-permissions.common/filter-schema-by-data-model-perms))]
                             (map :schema (f (map (fn [s] {:db_id id :schema s}) schemas)))
                             schemas)
                           (filter (partial can-read-schema? id) schemas)))]
    (if include_editable_data_model
      (api/check-404 (t2/select-one Database id))
      (api/read-check Database id))
    (->> (t2/select-fn-set :schema Table
                           :db_id id :active true
                           (merge
                            {:order-by [[:%lower.schema :asc]]}
                            (when-not include_hidden
                              ;; a non-nil value means Table is hidden -- see [[metabase.models.table/visibility-types]]
                              {:where [:= :visibility_type nil]})))
         filter-schemas
         ;; for `nil` schemas return the empty string
         (map #(if (nil? %) "" %))
         distinct
         sort)))

(api/defendpoint GET ["/:virtual-db/schemas"
                      :virtual-db (re-pattern (str lib.schema.id/saved-questions-virtual-database-id))]
  "Returns a list of all the schemas found for the saved questions virtual database."
  []
  (when (public-settings/enable-nested-queries)
    (->> (cards-virtual-tables :card)
         (map :schema)
         distinct
         (sort-by u/lower-case-en))))

(api/defendpoint GET ["/:virtual-db/datasets"
                      :virtual-db (re-pattern (str lib.schema.id/saved-questions-virtual-database-id))]
  "Returns a list of all the datasets found for the saved questions virtual database."
  []
  (when (public-settings/enable-nested-queries)
    (->> (cards-virtual-tables :dataset)
         (map :schema)
         distinct
         (sort-by u/lower-case-en))))

(defn- schema-tables-list
  ([db-id schema]
   (schema-tables-list db-id schema nil nil))
  ([db-id schema include_hidden include_editable_data_model]
   (when-not include_editable_data_model
     (api/read-check Database db-id)
     (api/check-403 (can-read-schema? db-id schema)))
   (let [tables (if include_hidden
                  (t2/select Table
                             :db_id db-id
                             :schema schema
                             :active true
                             {:order-by [[:display_name :asc]]})
                  (t2/select Table
                             :db_id db-id
                             :schema schema
                             :active true
                             :visibility_type nil
                             {:order-by [[:display_name :asc]]}))]
     (if include_editable_data_model
       (if-let [f (when config/ee-available?
                    (classloader/require 'metabase-enterprise.advanced-permissions.common)
                    (resolve 'metabase-enterprise.advanced-permissions.common/filter-tables-by-data-model-perms))]
         (f tables)
         tables)
       (filter mi/can-read? tables)))))

(api/defendpoint GET 
  "Returns a list of Tables for the given Database `id` and `schema`"
  [id include_hidden include_editable_data_model schema]
  {id                          ms/PositiveInt
   include_hidden              [:maybe ms/BooleanValue]
   include_editable_data_model [:maybe ms/BooleanValue]}
  (api/check-404 (seq (schema-tables-list
                       id
                       schema
                       include_hidden
                       include_editable_data_model))))

(api/defendpoint GET 
  "Return a list of Tables for a Database whose `schema` is `nil` or an empty string."
  [id include_hidden include_editable_data_model]
  {id                          ms/PositiveInt
   include_hidden              [:maybe ms/BooleanValue]
   include_editable_data_model [:maybe ms/BooleanValue]}
  (api/check-404 (seq (concat (schema-tables-list id nil include_hidden include_editable_data_model)
                              (schema-tables-list id "" include_hidden include_editable_data_model)))))

(api/defendpoint GET ["/:virtual-db/schema/:schema"
                      :virtual-db (re-pattern (str lib.schema.id/saved-questions-virtual-database-id))]
  "Returns a list of Tables for the saved questions virtual database."
  [schema]
  (when (public-settings/enable-nested-queries)
    (->> (source-query-cards
          :card
          :additional-constraints [(if (= schema (api.table/root-collection-schema-name))
                                     [:= :collection_id nil]
                                     [:in :collection_id (api/check-404 (not-empty (t2/select-pks-set Collection :name schema)))])])
         (map api.table/card->virtual-table))))

(api/defendpoint GET ["/:virtual-db/datasets/:schema"
                      :virtual-db (re-pattern (str lib.schema.id/saved-questions-virtual-database-id))]
  "Returns a list of Tables for the datasets virtual database."
  [schema]
  (when (public-settings/enable-nested-queries)
    (->> (source-query-cards
          :dataset
          :additional-constraints [(if (= schema (api.table/root-collection-schema-name))
                                     [:= :collection_id nil]
                                     [:in :collection_id (api/check-404 (not-empty (t2/select-pks-set Collection :name schema)))])])
         (map api.table/card->virtual-table))))

(api/define-routes)

(ns metabase.api.dataset
  (:require
   [cheshire.core :as json]
   [clojure.string :as str]
   [compojure.core :refer [POST]]
   [metabase.api.common :as api]
   [metabase.api.field :as api.field]
   [metabase.driver :as driver]
   [metabase.driver.util :as driver.u]
   [metabase.events :as events]
   [metabase.lib.schema.id :as lib.schema.id]
   [metabase.mbql.normalize :as mbql.normalize]
   [metabase.mbql.schema :as mbql.s]
   [metabase.models.card :refer [Card]]
   [metabase.models.database :as database :refer [Database]]
   [metabase.models.params.custom-values :as custom-values]
   [metabase.models.persisted-info :as persisted-info]
   [metabase.models.query :as query]
   [metabase.models.table :refer [Table]]
   [metabase.query-processor :as qp]
   [metabase.query-processor.middleware.constraints :as qp.constraints]
   [metabase.query-processor.middleware.permissions :as qp.perms]
   [metabase.query-processor.pivot :as qp.pivot]
   [metabase.query-processor.streaming :as qp.streaming]
   [metabase.query-processor.util :as qp.util]
   [metabase.shared.models.visualization-settings :as mb.viz]
   [metabase.util :as u]
   [metabase.util.i18n :refer [trs tru]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [steffan-westcott.clj-otel.api.trace.span :as span]
   [toucan2.core :as t2]))

(defn- query->source-card-id
  [outer-query]
  (when-let [source-card-id (qp.util/query->source-card-id outer-query)]
    (log/info (trs "Source query for this query is Card {0}" (pr-str source-card-id)))
    (api/read-check Card source-card-id)
    source-card-id))

(defn- run-query-async
  [{:keys [database], :as query}
   & {:keys [context export-format qp-runner]
      :or   {context       :ad-hoc
             export-format :api
             qp-runner     qp/process-query-and-save-with-max-results-constraints!}}]
  (span/with-span!
    {:name "run-query-async"}
    (when (and (not= (:type query) "internal")
               (not= database lib.schema.id/saved-questions-virtual-database-id))
      (when-not database
        (throw (ex-info (tru "`database` is required for all queries whose type is not `internal`.")
                        {:status-code 400, :query query})))
      (api/read-check Database database))
    ;; store table id trivially iff we get a query with simple source-table
    (let [table-id (get-in query [:query :source-table])]
      (when (int? table-id)
        (events/publish-event! :event/table-read {:object  (t2/select-one Table :id table-id)
                                                  :user-id api/*current-user-id*})))
    ;; add sensible constraints for results limits on our query
    (let [source-card-id (query->source-card-id query)
          source-card    (when source-card-id
                           (t2/select-one [Card :result_metadata :dataset] :id source-card-id))
          info           (cond-> {:executed-by api/*current-user-id*
                                  :context     context
                                  :card-id     source-card-id}
                           (:dataset source-card)
                           (assoc :metadata/dataset-metadata (:result_metadata source-card)))]
      (binding [qp.perms/*card-id* source-card-id]
        (qp.streaming/streaming-response [{:keys [rff context]} export-format]
                                         (qp-runner query info rff context))))))

(api/defendpoint POST 
  "Execute a query and retrieve the results in the usual format. The query will not use the cache."
  [:as {{:keys [database] :as query} :body}]
  {database [:maybe :int]}
  (run-query-async (update-in query [:middleware :js-int-to-string?] (fnil identity true))))

(def export-formats
  (mapv u/qualified-name (qp.streaming/export-formats)))

(def ExportFormat
  (into [:enum] export-formats))

(mu/defn export-format->context :- mbql.s/Context
  "Return the `:context` that should be used when saving a QueryExecution triggered by a request to download results
  in `export-format`.
    (export-format->context :json) ;-> :json-download"
  [export-format]
  (keyword (str (u/qualified-name export-format) "-download")))

(def export-format-regex
  (re-pattern (str "(" (str/join "|" (map u/qualified-name (qp.streaming/export-formats))) ")")))

(def ^:private column-ref-regex #"^\[.+\]$")

(defn- viz-setting-key-fn
   [json-key]
   (if (re-matches column-ref-regex json-key)
     json-key
     (keyword json-key)))

(api/defendpoint POST ["/:export-format", :export-format export-format-regex]
  "Execute a query and download the result data as a file in the specified format."
  [export-format :as {{:keys [query visualization_settings] :or {visualization_settings "{}"}} :params}]
  {query                  ms/JSONString
   visualization_settings ms/JSONString
   export-format          (into [:enum] export-formats)}
  (let [query        (json/parse-string query keyword)
        viz-settings (-> (json/parse-string visualization_settings viz-setting-key-fn)
                         (update :table.columns mbql.normalize/normalize)
                         mb.viz/db->norm)
        query        (-> (assoc query
                                :async? true
                                :viz-settings viz-settings)
                         (dissoc :constraints)
                         (update :middleware #(-> %
                                                  (dissoc :add-default-userland-constraints? :js-int-to-string?)
                                                  (assoc :process-viz-settings? true
                                                         :skip-results-metadata? true
                                                         :format-rows? false))))]
    (run-query-async
     query
     :export-format export-format
     :context       (export-format->context export-format)
     :qp-runner     qp/process-query-and-save-execution!)))

(api/defendpoint POST 
  "Get historical query execution duration."
  [:as {{:keys [database], :as query} :body}]
  (api/read-check Database database)
  ;; try calculating the average for the query as it was given to us, otherwise with the default constraints if
  ;; there's no data there. If we still can't find relevant info, just default to 0
  {:average (or
             (some (comp query/average-execution-time-ms qp.util/query-hash)
                   [query
                    (assoc query :constraints (qp.constraints/default-query-constraints))])
             0)})

(api/defendpoint POST 
  "Fetch a native version of an MBQL query."
  [:as {{:keys [database pretty] :as query} :body}]
  {database ms/PositiveInt
   pretty  [:maybe :boolean]}
  (binding [persisted-info/*allow-persisted-substitution* false]
    (qp.perms/check-current-user-has-adhoc-native-query-perms query)
    (let [driver (driver.u/database->driver database)
          prettify (partial driver/prettify-native-form driver)
          compiled (qp/compile-and-splice-parameters query)]
      (cond-> compiled
        (not (false? pretty)) (update :query prettify)))))

(api/defendpoint POST 
  "Generate a pivoted dataset for an ad-hoc query"
  [:as {{:keys [database] :as query} :body}]
  {database [:maybe ms/PositiveInt]}
  (when-not database
    (throw (Exception. (str (tru "`database` is required for all queries.")))))
  (api/read-check Database database)
  (let [info {:executed-by api/*current-user-id*
              :context     :ad-hoc}]
    (qp.streaming/streaming-response [{:keys [rff context]} :api]
      (qp.pivot/run-pivot-query (assoc query
                                       :async? true
                                       :constraints (qp.constraints/default-query-constraints))
                                info
                                rff
                                context))))

(defn- parameter-field-values
  [field-ids query]
  (when-not (seq field-ids)
    (throw (ex-info (tru "Missing field-ids for parameter")
                    {:status-code 400})))
  (-> (reduce (fn [resp id]
                (let [{values :values more? :has_more_values} (api.field/search-values-from-field-id id query)]
                  (-> resp
                      (update :values concat values)
                      (update :has_more_values #(or % more?)))))
              {:has_more_values false
               :values          []}
              field-ids)
      ;; deduplicate the values returned from multiple fields
      (update :values (comp vec set))))

(defn parameter-values
  [parameter field-ids query]
  (custom-values/parameter->values
    parameter query
    (fn [] (parameter-field-values field-ids query))))

(api/defendpoint POST 
  "Return parameter values for cards or dashboards that are being edited."
  [:as {{:keys [parameter field_ids]} :body}]
  {parameter ms/Parameter
   field_ids [:maybe [:sequential ms/PositiveInt]]}
  (parameter-values parameter field_ids nil))

(api/defendpoint POST 
  "Return parameter values for cards or dashboards that are being edited. Expects a query string at `?query=foo`."
  [query :as {{:keys [parameter field_ids]} :body}]
  {parameter ms/Parameter
   field_ids [:maybe [:sequential ms/PositiveInt]]
   query     ms/NonBlankString}
  (parameter-values parameter field_ids query))

(api/define-routes)

(ns metabase.api.email
  (:require
   [clojure.data :as data]
   [clojure.set :as set]
   [clojure.string :as str]
   [compojure.core :refer [DELETE POST PUT]]
   [metabase.api.common :as api]
   [metabase.api.common.validation :as validation]
   [metabase.email :as email]
   [metabase.models.setting :as setting]
   [metabase.util :as u]
   [metabase.util.i18n :refer [tru]]
   [metabase.util.log :as log]))

(set! *warn-on-reflection* true)

(def ^:private mb-to-smtp-settings
  {:email-smtp-host     :host
   :email-smtp-username :user
   :email-smtp-password :pass
   :email-smtp-port     :port
   :email-smtp-security :security
   :email-from-name     :sender-name
   :email-from-address  :sender
   :email-reply-to      :reply-to})

(defn- humanize-error-messages
  [{::email/keys [error]}]
  (when error
    (let [conn-error  {:errors {:email-smtp-host "Wrong host or port"
                                :email-smtp-port "Wrong host or port"}}
          creds-error {:errors {:email-smtp-username "Wrong username or password"
                                :email-smtp-password "Wrong username or password"}}
          exceptions  (u/full-exception-chain error)
          message     (str/join ": " (map ex-message exceptions))
          match-error (fn match-error [regex-or-exception-class [message exceptions]]
                        (cond (instance? java.util.regex.Pattern regex-or-exception-class)
                              (re-find regex-or-exception-class message)
                              (class? regex-or-exception-class)
                              (some (partial instance? regex-or-exception-class) exceptions)))]
      (log/warn "Problem connecting to mail server:" message)
      (condp match-error [message exceptions]
        ;; bad host = "Unknown SMTP host: foobar"
        #"^Unknown SMTP host:.*$"
        conn-error
        ;; host seems valid, but host/port failed connection = "Could not connect to SMTP host: localhost, port: 123"
        #".*Could(?: not)|(?:n't) connect to (?:SMTP )?host.*"
        conn-error
        ;; seen this show up on mandrill
        #"^Invalid Addresses$"
        creds-error
        ;; seen this show up on mandrill using TLS with bad credentials
        #"^failed to connect, no password specified\?$"
        creds-error
        ;; madrill authentication failure
        #"^435 4.7.8 Error: authentication failed:.*$"
        creds-error
        javax.mail.AuthenticationFailedException
        creds-error
        ;; everything else :(
        {:message (str "Sorry, something went wrong. Please try again. Error: " message)}))))

(defn- humanize-email-corrections
  [corrections]
  (into
   {}
   (for [[k v] corrections]
     [k (tru "{0} was autocorrected to {1}"
             (name (mb-to-smtp-settings k))
             (u/upper-case-en v))])))

(defn- env-var-values-by-email-setting
  []
  (into {}
        (for [setting-name (keys mb-to-smtp-settings)
              :let         [value (setting/env-var-value setting-name)]
              :when        (some? value)]
          [setting-name value])))

(api/defendpoint PUT 
  "Update multiple email Settings. You must be a superuser or have `setting` permission to do this."
  [:as {settings :body}]
  {settings :map}
  (validation/check-has-application-permission :setting)
  (let [;; the frontend has access to an obfuscated version of the password. Watch for whether it sent us a new password or
        ;; the obfuscated version
        obfuscated? (and (:email-smtp-password settings) (email/email-smtp-password)
                         (= (:email-smtp-password settings) (setting/obfuscate-value (email/email-smtp-password))))
        ;; override `nil` values in the request with environment variables for testing the SMTP connection
        env-var-settings (env-var-values-by-email-setting)
        settings         (merge settings env-var-settings)
        settings         (-> (cond-> settings
                               obfuscated?
                               (assoc :email-smtp-password (email/email-smtp-password)))
                             (select-keys (keys mb-to-smtp-settings))
                             (set/rename-keys mb-to-smtp-settings))
        settings         (cond-> settings
                           (string? (:port settings))     (update :port #(Long/parseLong ^String %))
                           (string? (:security settings)) (update :security keyword))
        response         (email/test-smtp-connection settings)]
    (if-not (::email/error response)
      ;; test was good, save our settings
      (let [[_ corrections] (data/diff settings response)
            new-settings    (set/rename-keys response (set/map-invert mb-to-smtp-settings))]
        ;; don't update settings if they are set by environment variables
        (setting/set-many! (apply dissoc new-settings (keys env-var-settings)))
        (cond-> (assoc new-settings :with-corrections (-> corrections
                                                          (set/rename-keys (set/map-invert mb-to-smtp-settings))
                                                          humanize-email-corrections))
          obfuscated? (update :email-smtp-password setting/obfuscate-value)))
      ;; test failed, return response message
      {:status 400
       :body   (humanize-error-messages response)})))

(api/defendpoint DELETE 
  "Clear all email related settings. You must be a superuser or have `setting` permission to do this."
  []
  (validation/check-has-application-permission :setting)
  (setting/set-many! (zipmap (keys mb-to-smtp-settings) (repeat nil)))
  api/generic-204-no-content)

(api/defendpoint POST 
  "Send a test email using the SMTP Settings. You must be a superuser or have `setting` permission to do this.
  Returns `{:ok true}` if we were able to send the message successfully, otherwise a standard 400 error response."
  []
  (validation/check-has-application-permission :setting)
  (let [response (email/send-message!
                   :subject      "Metabase Test Email"
                   :recipients   [(:email @api/*current-user*)]
                   :message-type :text
                   :message      "Your Metabase emails are working — hooray!")]
    (if-not (::email/error response)
      {:ok true}
      {:status 400
       :body   (humanize-error-messages response)})))

(api/define-routes)

(ns metabase.api.embed
  (:require
   [clojure.set :as set]
   [clojure.string :as str]
   [compojure.core :refer [GET]]
   [medley.core :as m]
   [metabase.api.card :as api.card]
   [metabase.api.common :as api]
   [metabase.api.common.validation :as validation]
   [metabase.api.dashboard :as api.dashboard]
   [metabase.api.dataset :as api.dataset]
   [metabase.api.public :as api.public]
   [metabase.driver.common.parameters.operators :as params.ops]
   [metabase.events :as events]
   [metabase.models.card :as card :refer [Card]]
   [metabase.models.dashboard :refer [Dashboard]]
   [metabase.models.params :as params]
   [metabase.pulse.parameters :as pulse-params]
   [metabase.query-processor :as qp]
   [metabase.query-processor.middleware.constraints :as qp.constraints]
   [metabase.query-processor.pivot :as qp.pivot]
   [metabase.util :as u]
   [metabase.util.embed :as embed]
   [metabase.util.i18n
    :as i18n
    :refer [tru]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [toucan2.core :as t2]))

(set! *warn-on-reflection* true)

(defn- check-params-are-allowed
  [object-embedding-params token-params user-params]
  (let [all-params        (set/union token-params user-params)
        duplicated-params (set/intersection token-params user-params)]
    (doseq [[param status] object-embedding-params]
      (case status
        ;; disabled means a param is not allowed to be specified by either token or user
        "disabled" (api/check (not (contains? all-params param))
                              [400 (tru "You''re not allowed to specify a value for {0}." param)])
        ;; enabled means either JWT *or* user can specify the param, but not both. Param is *not* required
        "enabled"  (api/check (not (contains? duplicated-params param))
                              [400 (tru "You can''t specify a value for {0} if it''s already set in the JWT." param)])
        ;; locked means JWT must specify param
        "locked"   (api/check
                    (contains? token-params param)      [400 (tru "You must specify a value for {0} in the JWT." param)]
                    (not (contains? user-params param)) [400 (tru "You can only specify a value for {0} in the JWT." param)])))))

(defn- check-params-exist
  [object-embedding-params all-params]
  (let [embedding-params (set (keys object-embedding-params))]
    (doseq [k all-params]
      (api/check (contains? embedding-params k)
        [400 (format "Unknown parameter %s." k)]))))

(defn- check-param-sets
  [object-embedding-params token-params user-params]
  ;; TODO - maybe make this log/debug once embedding is wrapped up
  (log/debug "Validating params for embedded object:\n"
             "object embedding params:" object-embedding-params
             "token params:"            token-params
             "user params:"             user-params)
  (check-params-are-allowed object-embedding-params token-params user-params)
  (check-params-exist object-embedding-params (set/union token-params user-params)))

(defn- valid-param?
  [v]
  (or (not (string? v))
      (not (str/blank? v))))

(mu/defn ^:private validate-and-merge-params :- [:map-of :keyword :any]
  "Validate that the `token-params` passed in the JWT and the `user-params` (passed as part of the URL) are allowed, and
  that ones that are required are specified by checking them against a Card or Dashboard's `object-embedding-params`
  (the object's value of `:embedding_params`). Throws a 400 if any of the checks fail. If all checks are successful,
  returns a *merged* parameters map."
  [object-embedding-params :- ms/EmbeddingParams
   token-params            :- [:map-of :keyword :any]
   user-params             :- [:map-of :keyword :any]]
  (check-param-sets object-embedding-params
                    (set (keys (m/filter-vals valid-param? token-params)))
                    (set (keys (m/filter-vals valid-param? user-params))))
  ;; ok, everything checks out, now return the merged params map
  (merge user-params token-params))

(defn- remove-params-in-set
  [params params-to-remove]
  (for [param params
        :when (not (contains? params-to-remove (keyword (:slug param))))]
    param))

(defn- get-params-to-remove
  [dashboard-or-card embedding-params]
  (set (concat (for [[param status] embedding-params
                     :when          (not= status "enabled")]
                 param)
               (for [{slug :slug} (:parameters dashboard-or-card)
                     :let         [param (keyword slug)]
                     :when        (not (contains? embedding-params param))]
                 param))))

(mu/defn ^:private remove-locked-and-disabled-params
  [dashboard-or-card embedding-params :- ms/EmbeddingParams]
  (let [params-to-remove (get-params-to-remove dashboard-or-card embedding-params)]
    (update dashboard-or-card :parameters remove-params-in-set params-to-remove)))

(defn- remove-token-parameters
  [dashboard-or-card token-params]
  (update dashboard-or-card :parameters remove-params-in-set (set (keys token-params))))

(defn- substitute-token-parameters-in-text
  [dashboard token-params]
  (let [params             (:parameters dashboard)
        dashcards      (:dashcards dashboard)
        params-with-values (reduce
                            (fn [acc param]
                             (if-let [value (get token-params (keyword (:slug param)))]
                                (conj acc (assoc param :value value))
                                acc))
                            []
                            params)]
    (assoc dashboard
           :dashcards
           (map
            (fn [card]
              (if (-> card :visualization_settings :virtual_card)
                (pulse-params/process-virtual-dashcard card params-with-values)
                card))
            dashcards))))

(mu/defn ^:private apply-slug->value :- [:maybe [:sequential
                                                 [:map
                                                  [:slug ms/NonBlankString]
                                                  [:type :keyword]
                                                  [:target :any]
                                                  [:value :any]]]]
  "Adds `value` to parameters with `slug` matching a key in `merged-slug->value` and removes parameters without a
   `value`."
  [parameters slug->value]
  (when (seq parameters)
    (for [param parameters
          :let  [slug  (keyword (:slug param))
                 value (get slug->value slug)
                 ;; operator parameters expect a sequence of values so if we get a lone value (e.g. from a single URL
                 ;; query parameter) wrap it in a sequence
                 value (if (and (some? value)
                                (params.ops/operator? (:type param)))
                         (u/one-or-many value)
                         value)]
          :when (contains? slug->value slug)]
      (assoc (select-keys param [:type :target :slug])
             :value value))))

(defn- resolve-card-parameters
   ; TODO - better docstring
  [card-or-id]
  (-> (t2/select-one [Card :dataset_query :parameters], :id (u/the-id card-or-id))
      api.public/combine-parameters-and-template-tags
      :parameters))

(mu/defn ^:private resolve-dashboard-parameters :- [:sequential api.dashboard/ParameterWithID]
  "Given a `dashboard-id` and parameters map in the format `slug->value`, return a sequence of parameters with `:id`s
  that can be passed to various functions in the `metabase.api.dashboard` namespace such as
  [[metabase.api.dashboard/run-query-for-dashcard-async]]."
  [dashboard-id :- ms/PositiveInt
   slug->value  :- :map]
  (let [parameters (t2/select-one-fn :parameters Dashboard :id dashboard-id)
        slug->id   (into {} (map (juxt :slug :id)) parameters)]
    (vec (for [[slug value] slug->value
               :let         [slug (u/qualified-name slug)]]
           {:slug  slug
            :id    (or (get slug->id slug)
                       (throw (ex-info (tru "No matching parameter with slug {0}. Found: {1}" (pr-str slug) (pr-str (keys slug->id)))
                                       {:status-code          400
                                        :slug                 slug
                                        :dashboard-parameters parameters})))
            :value value}))))

(mu/defn ^:private normalize-query-params :- [:map-of :keyword :any]
  "Take a map of `query-params` and make sure they're in the right format for the rest of our code. Our
  `wrap-keyword-params` middleware normally converts all query params keys to keywords, but only if they seem like
  ones that make sense as keywords. Some params, such as ones that start with a number, do not pass this test, and are
  not automatically converted. Thus we must do it ourselves here to make sure things are done as we'd expect.
  Also, any param values that are blank strings should be parsed as nil, representing the absence of a value."
  [query-params]
  (-> query-params
      (update-keys keyword)
      (update-vals (fn [v] (if (= v ) nil v)))))

(defn card-for-unsigned-token
  [unsigned-token & {:keys [embedding-params constraints]}]
  {:pre [((some-fn empty? sequential?) constraints) (even? (count constraints))]}
  (let [card-id      (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :question])
        token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])]
    (-> (apply api.public/public-card :id card-id, constraints)
        api.public/combine-parameters-and-template-tags
        (remove-token-parameters token-params)
        (remove-locked-and-disabled-params (or embedding-params
                                               (t2/select-one-fn :embedding_params Card :id card-id))))))

(defn run-query-for-card-with-params-async
  {:style/indent 0}
  [& {:keys [export-format card-id embedding-params token-params query-params qp-runner constraints options]
      :or   {qp-runner qp/process-query-and-save-execution!}}]
  {:pre [(integer? card-id) (u/maybe? map? embedding-params) (map? token-params) (map? query-params)]}
  (let [merged-slug->value (validate-and-merge-params embedding-params token-params (normalize-query-params query-params))
        parameters         (apply-slug->value (resolve-card-parameters card-id) merged-slug->value)]
    (m/mapply api.public/run-query-for-card-with-id-async
              card-id export-format parameters
              :context :embedded-question,
              :constraints constraints,
              :qp-runner qp-runner,
              options)))

(defn- remove-linked-filters-param-values [dashboard]
  (let [param-ids (set (map :id (:parameters dashboard)))
        param-ids-to-remove (set (for [{param-id :id
                                        filtering-parameters :filteringParameters} (:parameters dashboard)
                                       filtering-parameter-id filtering-parameters
                                       :when (not (contains? param-ids filtering-parameter-id))]
                                   param-id))
        linked-field-ids (set (mapcat (params/get-linked-field-ids (:dashcards dashboard)) param-ids-to-remove))]
    (update dashboard :param_values #(->> %
                                          (map (fn [[param-id param]]
                                                 {param-id (cond-> param
                                                             (contains? linked-field-ids param-id) ;; is param linked?
                                                             (assoc :values []))}))
                                          (into {})))))

(defn- remove-locked-parameters [dashboard embedding-params]
  (let [params-to-remove (get-params-to-remove dashboard embedding-params)
        param-ids-to-remove (set (for [parameter (:parameters dashboard)
                                       :when     (contains? params-to-remove (keyword (:slug parameter)))]
                                   (:id parameter)))
        linked-field-ids (set (mapcat (params/get-linked-field-ids (:dashcards dashboard)) param-ids-to-remove))
        remove-parameters (fn [dashcard]
                            (update dashcard :parameter_mappings
                                    (fn [param-mappings]
                                      (remove (fn [{:keys [parameter_id]}]
                                                (contains? param-ids-to-remove parameter_id)) param-mappings))))]
    (-> dashboard
        (update :dashcards #(map remove-parameters %))
        (update :param_fields #(apply dissoc % linked-field-ids))
        (update :param_values #(apply dissoc % linked-field-ids)))))

(defn dashboard-for-unsigned-token
  [unsigned-token & {:keys [embedding-params constraints]}]
  {:pre [((some-fn empty? sequential?) constraints) (even? (count constraints))]}
  (let [dashboard-id (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :dashboard])
        embedding-params (or embedding-params
                             (t2/select-one-fn :embedding_params Dashboard, :id dashboard-id))
        token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])]
    (-> (apply api.public/public-dashboard :id dashboard-id, constraints)
        (substitute-token-parameters-in-text token-params)
        (remove-locked-parameters embedding-params)
        (remove-token-parameters token-params)
        (remove-locked-and-disabled-params embedding-params)
        (remove-linked-filters-param-values))))

(defn- get-embed-dashboard-context
  [export-format]
  (case export-format
    "csv"  :embedded-csv-download
    "xlsx" :embedded-xlsx-download
    "json" :embedded-json-download
    :embedded-dashboard))

(defn dashcard-results-async
  {:style/indent 0}
  [& {:keys [dashboard-id dashcard-id card-id export-format embedding-params token-params middleware
             query-params constraints qp-runner]
      :or   {constraints (qp.constraints/default-query-constraints)
             qp-runner   qp/process-query-and-save-execution!}}]
  {:pre [(integer? dashboard-id) (integer? dashcard-id) (integer? card-id) (u/maybe? map? embedding-params)
         (map? token-params) (map? query-params)]}
  (let [slug->value (validate-and-merge-params embedding-params token-params (normalize-query-params query-params))
        parameters  (resolve-dashboard-parameters dashboard-id slug->value)]
    (api.public/public-dashcard-results-async
     :dashboard-id  dashboard-id
     :card-id       card-id
     :dashcard-id   dashcard-id
     :export-format export-format
     :parameters    parameters
     :qp-runner     qp-runner
     :context       (get-embed-dashboard-context export-format)
     :constraints   constraints
     :middleware    middleware)))

(defn- check-embedding-enabled-for-object
  ([entity id]
   (api/check (pos-int? id)
     [400 (tru "{0} id should be a positive integer." (name entity))])
   (check-embedding-enabled-for-object (t2/select-one [entity :enable_embedding] :id id)))
  ([object]
   (validation/check-embedding-enabled)
   (api/check-404 object)
   (api/check-not-archived object)
   (api/check (:enable_embedding object)
     [400 (tru "Embedding is not enabled for this object.")])))

(def ^:private ^{:arglists '([dashboard-id])} check-embedding-enabled-for-dashboard
  (partial check-embedding-enabled-for-object Dashboard))

(def ^:private ^{:arglists '([card-id])} check-embedding-enabled-for-card
  (partial check-embedding-enabled-for-object Card))

(api/defendpoint GET 
  "Fetch a Card via a JSON Web Token signed with the `embedding-secret-key`.
   Token should have the following format:
     {:resource {:question <card-id>}}"
  [token]
  (let [unsigned (embed/unsign token)]
    (check-embedding-enabled-for-card (embed/get-in-unsigned-token-or-throw unsigned [:resource :question]))
    (card-for-unsigned-token unsigned, :constraints [:enable_embedding true])))

(defn ^:private run-query-for-unsigned-token-async
  [unsigned-token export-format query-params & {:keys [constraints qp-runner]
                                                :or   {constraints (qp.constraints/default-query-constraints)
                                                       qp-runner   qp/process-query-and-save-execution!}
                                                :as   options}]
  (let [card-id (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :question])]
    (check-embedding-enabled-for-card card-id)
    (run-query-for-card-with-params-async
      :export-format     export-format
      :card-id           card-id
      :token-params      (embed/get-in-unsigned-token-or-throw unsigned-token [:params])
      :embedding-params  (t2/select-one-fn :embedding_params Card :id card-id)
      :query-params      query-params
      :qp-runner         qp-runner
      :constraints       constraints
      :options           options)))

(api/defendpoint GET 
  "Fetch the results of running a Card using a JSON Web Token signed with the `embedding-secret-key`.
   Token should have the following format:
     {:resource {:question <card-id>}
      :params   <parameters>}"
  [token & query-params]
  (run-query-for-unsigned-token-async (embed/unsign token) :api query-params))

(api/defendpoint GET ["/card/:token/query/:export-format", :export-format api.dataset/export-format-regex]
  "Like `GET /api/embed/card/query`, but returns the results as a file in the specified format."
  [token export-format :as {:keys [query-params]}]
  {export-format (into [:enum] api.dataset/export-formats)}
  (run-query-for-unsigned-token-async
   (embed/unsign token)
   export-format
   (m/map-keys keyword query-params)
   :constraints nil
   :middleware {:process-viz-settings? true
                :js-int-to-string?     false
                :format-rows?          false}))

(api/defendpoint GET 
  "Fetch a Dashboard via a JSON Web Token signed with the `embedding-secret-key`.
   Token should have the following format:
     {:resource {:dashboard <dashboard-id>}}"
  [token]
  (let [unsigned (embed/unsign token)]
    (check-embedding-enabled-for-dashboard (embed/get-in-unsigned-token-or-throw unsigned [:resource :dashboard]))
    (u/prog1 (dashboard-for-unsigned-token unsigned, :constraints [:enable_embedding true])
      (events/publish-event! :event/dashboard-read {:user-id api/*current-user-id*
                                                    :object <>}))))

(defn- dashcard-results-for-signed-token-async
  {:style/indent 1}
  [token dashcard-id card-id export-format query-params
   & {:keys [constraints qp-runner middleware]
      :or   {constraints (qp.constraints/default-query-constraints)
             qp-runner   qp/process-query-and-save-execution!}}]
  (let [unsigned-token (embed/unsign token)
        dashboard-id   (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :dashboard])]
    (check-embedding-enabled-for-dashboard dashboard-id)
    (dashcard-results-async
      :export-format    export-format
      :dashboard-id     dashboard-id
      :dashcard-id      dashcard-id
      :card-id          card-id
      :embedding-params (t2/select-one-fn :embedding_params Dashboard :id dashboard-id)
      :token-params     (embed/get-in-unsigned-token-or-throw unsigned-token [:params])
      :query-params     query-params
      :constraints      constraints
      :qp-runner        qp-runner
      :middleware       middleware)))

(api/defendpoint GET 
  "Fetch the results of running a Card belonging to a Dashboard using a JSON Web Token signed with the
  `embedding-secret-key`"
  [token dashcard-id card-id & query-params]
  {dashcard-id ms/PositiveInt
   card-id     ms/PositiveInt}
  (dashcard-results-for-signed-token-async token dashcard-id card-id :api query-params))

(api/defendpoint GET 
  "Fetch FieldValues for a Field that is referenced by an embedded Card."
  [token field-id]
  {field-id ms/PositiveInt}
  (let [unsigned-token (embed/unsign token)
        card-id        (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :question])]
    (check-embedding-enabled-for-card card-id)
    (api.public/card-and-field-id->values card-id field-id)))

(api/defendpoint GET 
  "Fetch FieldValues for a Field that is used as a param in an embedded Dashboard."
  [token field-id]
  {field-id ms/PositiveInt}
  (let [unsigned-token (embed/unsign token)
        dashboard-id   (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :dashboard])]
    (check-embedding-enabled-for-dashboard dashboard-id)
    (api.public/dashboard-and-field-id->values dashboard-id field-id)))

(api/defendpoint GET 
  "Search for values of a Field that is referenced by an embedded Card."
  [token field-id search-field-id value limit]
  {field-id        ms/PositiveInt
   search-field-id ms/PositiveInt
   value           ms/NonBlankString
   limit           [:maybe ms/PositiveInt]}
  (let [unsigned-token (embed/unsign token)
        card-id        (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :question])]
    (check-embedding-enabled-for-card card-id)
    (api.public/search-card-fields card-id field-id search-field-id value (when limit (Integer/parseInt limit)))))

(api/defendpoint GET 
  "Search for values of a Field that is referenced by a Card in an embedded Dashboard."
  [token field-id search-field-id value limit]
  {field-id        ms/PositiveInt
   search-field-id ms/PositiveInt
   value           ms/NonBlankString
   limit           [:maybe ms/PositiveInt]}
  (let [unsigned-token (embed/unsign token)
        dashboard-id   (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :dashboard])]
    (check-embedding-enabled-for-dashboard dashboard-id)
    (api.public/search-dashboard-fields dashboard-id field-id search-field-id value (when limit
                                                                                      (Integer/parseInt limit)))))

(api/defendpoint GET 
  "Fetch remapped Field values. This is the same as `GET /api/field/:id/remapping/:remapped-id`, but for use with
  embedded Cards."
  [token field-id remapped-id value]
  {field-id    ms/PositiveInt
   remapped-id ms/PositiveInt
   value       ms/NonBlankString}
  (let [unsigned-token (embed/unsign token)
        card-id        (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :question])]
    (check-embedding-enabled-for-card card-id)
    (api.public/card-field-remapped-values card-id field-id remapped-id value)))

(api/defendpoint GET 
  "Fetch remapped Field values. This is the same as `GET /api/field/:id/remapping/:remapped-id`, but for use with
  embedded Dashboards."
  [token field-id remapped-id value]
  {field-id    ms/PositiveInt
   remapped-id ms/PositiveInt
   value       ms/NonBlankString}
  (let [unsigned-token (embed/unsign token)
        dashboard-id   (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :dashboard])]
    (check-embedding-enabled-for-dashboard dashboard-id)
    (api.public/dashboard-field-remapped-values dashboard-id field-id remapped-id value)))

(api/defendpoint GET ["/dashboard/:token/dashcard/:dashcard-id/card/:card-id/:export-format"
                                         :export-format api.dataset/export-format-regex]
  "Fetch the results of running a Card belonging to a Dashboard using a JSON Web Token signed with the
  `embedding-secret-key` return the data in one of the export formats"
  [token export-format dashcard-id card-id, :as {:keys [query-params]}]
  {dashcard-id   ms/PositiveInt
   card-id       ms/PositiveInt
   export-format (into [:enum] api.dataset/export-formats)}
  (dashcard-results-for-signed-token-async token
    dashcard-id
    card-id
    export-format
    (m/map-keys keyword query-params)
    :constraints nil
    :middleware {:process-viz-settings? true
                 :js-int-to-string?     false
                 :format-rows?          false}))

(mu/defn ^:private param-values-merged-params :- [:map-of ms/NonBlankString :any]
  [id->slug slug->id embedding-params token-params id-query-params]
  (let [slug-query-params  (into {}
                                 (for [[id v] id-query-params]
                                   [(or (get id->slug (name id))
                                        (throw (ex-info (tru "Invalid query params: could not determine slug for parameter with ID {0}"
                                                             (pr-str id))
                                                        {:id              (name id)
                                                         :id->slug        id->slug
                                                         :id-query-params id-query-params})))
                                    v]))
        slug-query-params  (normalize-query-params slug-query-params)
        merged-slug->value (validate-and-merge-params embedding-params token-params slug-query-params)]
    (into {} (for [[slug value] merged-slug->value]
               [(get slug->id (name slug)) value]))))

(defn card-param-values
  [{:keys [unsigned-token card param-key search-prefix]}]
  (let [slug-token-params   (embed/get-in-unsigned-token-or-throw unsigned-token [:params])
        parameters          (or (seq (:parameters card))
                                (card/template-tag-parameters card))
        id->slug            (into {} (map (juxt :id :slug) parameters))
        slug->id            (into {} (map (juxt :slug :id) parameters))
        searched-param-slug (get id->slug param-key)
        embedding-params    (:embedding_params card)]
    (try
      (when-not (= (get embedding-params (keyword searched-param-slug)) "enabled")
        (throw (ex-info (tru "Cannot search for values: {0} is not an enabled parameter."
                             (pr-str searched-param-slug))
                        {:status-code 400})))
      (when (get slug-token-params (keyword searched-param-slug))
        (throw (ex-info (tru "You can''t specify a value for {0} if it's already set in the JWT." (pr-str searched-param-slug))
                        {:status-code 400})))
      (try
        (binding [api/*current-user-permissions-set* (atom #{"/"})]
          (api.card/param-values card param-key search-prefix))
        (catch Throwable e
          (throw (ex-info (.getMessage e)
                          {:card-id       (u/the-id card)
                           :param-key     param-key
                           :search-prefix search-prefix}
                          e))))
      (catch Throwable e
        (let [e (ex-info (.getMessage e)
                         {:card-id (u/the-id card)
                          :card-params (:parametres card)
                          :allowed-param-slugs embedding-params
                          :slug->id            slug->id
                          :id->slug            id->slug
                          :param-id            param-key
                          :param-slug          searched-param-slug
                          :token-params        slug-token-params}
                         e)]
          (log/errorf e "embedded card-param-values error\n%s"
                      (u/pprint-to-str (u/all-ex-data e)))
          (throw e))))))

(defn- dashboard-param-values [token searched-param-id prefix id-query-params]
  (let [unsigned-token                       (embed/unsign token)
        dashboard-id                         (embed/get-in-unsigned-token-or-throw unsigned-token [:resource :dashboard])
        _                                    (check-embedding-enabled-for-dashboard dashboard-id)
        slug-token-params                    (embed/get-in-unsigned-token-or-throw unsigned-token [:params])
        {parameters       :parameters
         embedding-params :embedding_params} (t2/select-one Dashboard :id dashboard-id)
        id->slug                             (into {} (map (juxt :id :slug) parameters))
        slug->id                             (into {} (map (juxt :slug :id) parameters))
        searched-param-slug                  (get id->slug searched-param-id)]
    (try
      ;; you can only search for values of a parameter if it is ENABLED and NOT PRESENT in the JWT.
      (when-not (= (get embedding-params (keyword searched-param-slug)) "enabled")
        (throw (ex-info (tru "Cannot search for values: {0} is not an enabled parameter." (pr-str searched-param-slug))
                        {:status-code 400})))
      (when (get slug-token-params (keyword searched-param-slug))
        (throw (ex-info (tru "You can''t specify a value for {0} if it's already set in the JWT." (pr-str searched-param-slug))
                        {:status-code 400})))
      ;; ok, at this point we can run the query
      (let [merged-id-params (param-values-merged-params id->slug slug->id embedding-params slug-token-params id-query-params)]
        (try
          (binding [api/*current-user-permissions-set* (atom #{"/"})]
            (api.dashboard/param-values (t2/select-one Dashboard :id dashboard-id) searched-param-id merged-id-params prefix))
          (catch Throwable e
            (throw (ex-info (.getMessage e)
                            {:merged-id-params merged-id-params}
                            e)))))
      (catch Throwable e
        (let [e (ex-info (.getMessage e)
                         {:dashboard-id        dashboard-id
                          :dashboard-params    parameters
                          :allowed-param-slugs embedding-params
                          :slug->id            slug->id
                          :id->slug            id->slug
                          :param-id            searched-param-id
                          :param-slug          searched-param-slug
                          :token-params        slug-token-params}
                         e)]
          (log/errorf e "Chain filter error\n%s" (u/pprint-to-str (u/all-ex-data e)))
          (throw e))))))

(api/defendpoint GET 
  "Embedded version of chain filter values endpoint."
  [token param-key :as {:keys [query-params]}]
  (dashboard-param-values token param-key nil query-params))

(api/defendpoint GET 
  "Embedded version of chain filter search endpoint."
  [token param-key prefix :as {:keys [query-params]}]
  (dashboard-param-values token param-key prefix query-params))

(api/defendpoint GET 
  "Embedded version of api.card filter values endpoint."
  [token param-key]
  (let [unsigned (embed/unsign token)
        card-id  (embed/get-in-unsigned-token-or-throw unsigned [:resource :question])
        card     (t2/select-one Card :id card-id)]
    (check-embedding-enabled-for-card card-id)
    (card-param-values {:unsigned-token unsigned
                        :card           card
                        :param-key      param-key})))

(api/defendpoint GET 
  "Embedded version of chain filter search endpoint."
  [token param-key prefix]
  (let [unsigned (embed/unsign token)
        card-id  (embed/get-in-unsigned-token-or-throw unsigned [:resource :question])
        card     (t2/select-one Card :id card-id)]
    (check-embedding-enabled-for-card card-id)
    (card-param-values {:unsigned-token unsigned
                        :card           card
                        :param-key      param-key
                        :search-prefix  prefix})))

(api/defendpoint GET 
  "Fetch the results of running a Card using a JSON Web Token signed with the `embedding-secret-key`.
   Token should have the following format:
     {:resource {:question <card-id>}
      :params   <parameters>}"
  [token & query-params]
  (run-query-for-unsigned-token-async (embed/unsign token) :api query-params :qp-runner qp.pivot/run-pivot-query))

(api/defendpoint GET 
  "Fetch the results of running a Card belonging to a Dashboard using a JSON Web Token signed with the
  `embedding-secret-key`"
  [token dashcard-id card-id & query-params]
  {dashcard-id ms/PositiveInt
   card-id ms/PositiveInt}
  (dashcard-results-for-signed-token-async token dashcard-id card-id :api query-params :qp-runner qp.pivot/run-pivot-query))

(api/define-routes)

(ns metabase.api.field
  (:require
   [clojure.string :as str]
   [compojure.core :refer [DELETE GET POST PUT]]
   [metabase.api.common :as api]
   [metabase.db.metadata-queries :as metadata-queries]
   [metabase.lib.schema.metadata :as lib.schema.metadata]
   [metabase.models.dimension :refer [Dimension]]
   [metabase.models.field :as field :refer [Field]]
   [metabase.models.field-values :as field-values :refer [FieldValues]]
   [metabase.models.interface :as mi]
   [metabase.models.params.chain-filter :as chain-filter]
   [metabase.models.params.field-values :as params.field-values]
   [metabase.models.permissions :as perms]
   [metabase.models.table :as table :refer [Table]]
   [metabase.query-processor :as qp]
   [metabase.related :as related]
   [metabase.server.middleware.offset-paging :as mw.offset-paging]
   [metabase.sync :as sync]
   [metabase.sync.concurrent :as sync.concurrent]
   [metabase.types :as types]
   [metabase.util :as u]
   [metabase.util.i18n :refer [trs]]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.schema :as ms]
   [toucan2.core :as t2])
  (:import
   (java.text NumberFormat)))

(set! *warn-on-reflection* true)

(def ^:private default-max-field-search-limit 1000)

(def ^:private FieldVisibilityType
  (into [:enum] (map name field/visibility-types)))

(defn- has-segmented-query-permissions?
  [table]
  (perms/set-has-full-permissions? @api/*current-user-permissions-set*
    (perms/table-sandboxed-query-path table)))

(defn- throw-if-no-read-or-segmented-perms
  [field]
  (when-not (or (mi/can-read? field)
                (has-segmented-query-permissions? (field/table field)))
    (api/throw-403)))

(api/defendpoint GET 
  "Get `Field` with ID."
  [id include_editable_data_model]
  {id                          ms/PositiveInt
   include_editable_data_model ms/BooleanValue}
  (let [field                       (-> (api/check-404 (t2/select-one Field :id id))
                                        (t2/hydrate [:table :db] :has_field_values :dimensions :name_field))
        field                       (if include_editable_data_model
                                      (field/hydrate-target-with-write-perms field)
                                      (t2/hydrate field :target))]
    ;; Normal read perms = normal access.
    ;;
    ;; There's also a special case where we allow you to fetch a Field even if you don't have full read permissions for
    ;; it: if you have segmented query access to the Table it belongs to. In this case, we'll still let you fetch the
    ;; Field, since this is required to power features like Dashboard filters, but we'll treat this Field a little
    ;; differently in other endpoints such as the FieldValues fetching endpoint.
    ;;
    ;; Check for permissions and throw 403 if we don't have them...
    (if include_editable_data_model
      (api/write-check Table (:table_id field))
      (throw-if-no-read-or-segmented-perms field))
    ;; ...but if we do, return the Field <3
    field))

(defn- clear-dimension-on-fk-change! [{:keys [dimensions], :as _field}]
  (doseq [{dimension-id :id, dimension-type :type} dimensions]
    (when (and dimension-id (= :external dimension-type))
      (t2/delete! Dimension :id dimension-id))))

(defn- removed-fk-semantic-type? [old-semantic-type new-semantic-type]
  (and (not= old-semantic-type new-semantic-type)
       (isa? old-semantic-type :type/FK)
       (or (nil? new-semantic-type)
           (not (isa? new-semantic-type :type/FK)))))

(defn- internal-remapping-allowed? [base-type semantic-type]
  (and (isa? base-type :type/Integer)
       (or
        (nil? semantic-type)
        (isa? semantic-type :type/Category)
        (isa? semantic-type :type/Enum))))

(defn- clear-dimension-on-type-change!
  [{:keys [dimensions], :as _old-field} base-type new-semantic-type]
  (doseq [{old-dim-id :id, old-dim-type :type} dimensions]
    (when (and old-dim-id
               (= :internal old-dim-type)
               (not (internal-remapping-allowed? base-type new-semantic-type)))
      (t2/delete! Dimension :id old-dim-id))))

(defn- update-nested-fields-on-json-unfolding-change!
  [old-field new-json-unfolding]
  (when (not= new-json-unfolding (:json_unfolding old-field))
    (if new-json-unfolding
      (let [update-result (t2/update! Field
                                      :table_id (:table_id old-field)
                                      :nfc_path [:like (str "[\"" (:name old-field) "\",%]")]
                                      {:active true})]
        (when (zero? update-result)
          ;; Sync the table if no nested fields exist. This means the table hasn't previously
          ;; been synced when JSON unfolding was enabled. This assumes the JSON field is already updated to have
          ;; JSON unfolding enabled.
          (let [table (field/table old-field)]
            (sync.concurrent/submit-task (fn [] (sync/sync-table! table))))))
      (t2/update! Field
                  :table_id (:table_id old-field)
                  :nfc_path [:like (str "[\"" (:name old-field) "\",%]")]
                  {:active false})))
  nil)

(api/defendpoint PUT 
  "Update `Field` with ID."
  [id :as {{:keys [caveats description display_name fk_target_field_id points_of_interest semantic_type
                   coercion_strategy visibility_type has_field_values settings nfc_path json_unfolding]
            :as   body} :body}]
  {id                 ms/PositiveInt
   caveats            [:maybe ms/NonBlankString]
   description        [:maybe ms/NonBlankString]
   display_name       [:maybe ms/NonBlankString]
   fk_target_field_id [:maybe ms/PositiveInt]
   points_of_interest [:maybe ms/NonBlankString]
   semantic_type      [:maybe ms/FieldSemanticOrRelationTypeKeywordOrString]
   coercion_strategy  [:maybe ms/CoercionStrategyKeywordOrString]
   visibility_type    [:maybe FieldVisibilityType]
   has_field_values   [:maybe ::lib.schema.metadata/column.has-field-values]
   settings           [:maybe ms/Map]
   nfc_path           [:maybe [:sequential ms/NonBlankString]]
   json_unfolding     [:maybe :boolean]}
  (let [field             (t2/hydrate (api/write-check Field id) :dimensions)
        new-semantic-type (keyword (get body :semantic_type (:semantic_type field)))
        [effective-type coercion-strategy]
        (or (when-let [coercion_strategy (keyword coercion_strategy)]
              (let [effective (types/effective-type-for-coercion coercion_strategy)]
                ;; throw an error in an else branch?
                (when (types/is-coercible? coercion_strategy (:base_type field) effective)
                  [effective coercion_strategy])))
            [(:base_type field) nil])
        removed-fk?        (removed-fk-semantic-type? (:semantic_type field) new-semantic-type)
        fk-target-field-id (get body :fk_target_field_id (:fk_target_field_id field))]
    ;; validate that fk_target_field_id is a valid Field
    ;; TODO - we should also check that the Field is within the same database as our field
    (when fk-target-field-id
      (api/checkp (t2/exists? Field :id fk-target-field-id)
        :fk_target_field_id "Invalid target field"))
    (when (and display_name
               (not removed-fk?)
               (not= (:display_name field) display_name))
      (t2/update! Dimension :field_id id {:name display_name}))
    ;; everything checks out, now update the field
    (api/check-500
     (t2/with-transaction [_conn]
       (when removed-fk?
         (clear-dimension-on-fk-change! field))
       (clear-dimension-on-type-change! field (:base_type field) new-semantic-type)
       (t2/update! Field id
                   (u/select-keys-when (assoc body
                                              :fk_target_field_id (when-not removed-fk? fk-target-field-id)
                                              :effective_type effective-type
                                              :coercion_strategy coercion-strategy)
                                       :present #{:caveats :description :fk_target_field_id :points_of_interest :semantic_type :visibility_type
                                                  :coercion_strategy :effective_type :has_field_values :nfc_path :json_unfolding}
                                       :non-nil #{:display_name :settings}))))
    (when (some? json_unfolding)
      (update-nested-fields-on-json-unfolding-change! field json_unfolding))
    ;; return updated field. note the fingerprint on this might be out of date if the task below would replace them
    ;; but that shouldn't matter for the datamodel page
    (u/prog1 (-> (t2/select-one Field :id id)
                 (t2/hydrate :dimensions :has_field_values)
                 (field/hydrate-target-with-write-perms))
      (when (not= effective-type (:effective_type field))
        (sync.concurrent/submit-task (fn [] (sync/refingerprint-field! <>)))))))

(api/defendpoint GET 
  "Get the count and distinct count of `Field` with ID."
  [id]
  {id ms/PositiveInt}
  (let [field (api/read-check Field id)]
    [[:count     (metadata-queries/field-count field)]
     [:distincts (metadata-queries/field-distinct-count field)]]))

(api/defendpoint POST 
  "Sets the dimension for the given field at ID"
  [id :as {{dimension-type :type, dimension-name :name, human_readable_field_id :human_readable_field_id} :body}]
  {id                      ms/PositiveInt
   dimension-type          [:enum "internal" "external"]
   dimension-name          ms/NonBlankString
   human_readable_field_id [:maybe ms/PositiveInt]}
  (api/write-check Field id)
  (api/check (or (= dimension-type "internal")
                 (and (= dimension-type "external")
                      human_readable_field_id))
             [400 "Foreign key based remappings require a human readable field id"])
  (if-let [dimension (t2/select-one Dimension :field_id id)]
    (t2/update! Dimension (u/the-id dimension)
                {:type                    dimension-type
                 :name                    dimension-name
                 :human_readable_field_id human_readable_field_id})
    (t2/insert! Dimension
                {:field_id                id
                 :type                    dimension-type
                 :name                    dimension-name
                 :human_readable_field_id human_readable_field_id}))
  (t2/select-one Dimension :field_id id))

(api/defendpoint DELETE 
  "Remove the dimension associated to field at ID"
  [id]
  {id ms/PositiveInt}
  (api/write-check Field id)
  (t2/delete! Dimension :field_id id)
  api/generic-204-no-content)

(def ^:private empty-field-values
  {:values []})

(declare search-values)

(mu/defn field->values :- ms/FieldValuesResult
  "Fetch FieldValues, if they exist, for a `field` and return them in an appropriate format for public/embedded
  use-cases."
  [{has-field-values-type :has_field_values, field-id :id, has_more_values :has_more_values, :as field}]
  ;; TODO: explain why using remapped fields is restricted to `has_field_values=list`
  (if-let [remapped-field-id (when (= has-field-values-type :list)
                               (chain-filter/remapped-field-id field-id))]
    {:values          (search-values (api/check-404 field)
                                     (api/check-404 (t2/select-one Field :id remapped-field-id)))
     :field_id        field-id
     :has_more_values (boolean has_more_values)}
    (params.field-values/get-or-create-field-values-for-current-user! (api/check-404 field))))

(mu/defn search-values-from-field-id :- ms/FieldValuesResult
  "Search for values of a field given by `field-id` that contain `query`."
  [field-id query]
  (let [field        (api/read-check (t2/select-one Field :id field-id))
        search-field (or (some->> (chain-filter/remapped-field-id field-id)
                                  (t2/select-one Field :id))
                         field)]
    {:values          (search-values field search-field query)
     ;; assume there are more if doing a search, otherwise there are no more values
     :has_more_values (not (str/blank? query))
     :field_id        field-id}))

(api/defendpoint GET 
  "If a Field's value of `has_field_values` is `:list`, return a list of all the distinct values of the Field (or
  remapped Field), and (if defined by a User) a map of human-readable remapped values. If `has_field_values` is not
  `:list`, checks whether we should create FieldValues for this Field; if so, creates and returns them."
  [id]
  {id ms/PositiveInt}
  (let [field (api/read-check (t2/select-one Field :id id))]
    (field->values field)))

(api/defendpoint GET 
  "Implementation of the field values endpoint for fields in the Saved Questions 'virtual' DB. This endpoint is just a
  convenience to simplify the frontend code. It just returns the standard 'empty' field values response."
  ;; we don't actually care what field-name or field-type are, so they're ignored
  [_ _]
  empty-field-values)

(defn- validate-human-readable-pairs
  [value-pairs]
  (let [human-readable-missing? #(= ::not-found (get % 1 ::not-found))
        has-human-readable-values? (not-any? human-readable-missing? value-pairs)]
    (api/check (or has-human-readable-values?
                   (every? human-readable-missing? value-pairs))
      [400 "If remapped values are specified, they must be specified for all field values"])
    has-human-readable-values?))

(defn- update-field-values! [field-value-id value-pairs]
  (let [human-readable-values? (validate-human-readable-pairs value-pairs)]
    (api/check-500 (pos? (t2/update! FieldValues field-value-id
                                     {:values (map first value-pairs)
                                      :human_readable_values (when human-readable-values?
                                                               (map second value-pairs))})))))

(defn- create-field-values!
  [field-or-id value-pairs]
  (let [human-readable-values? (validate-human-readable-pairs value-pairs)]
    (t2/insert! FieldValues
                :type :full
                :field_id (u/the-id field-or-id)
                :values (map first value-pairs)
                :human_readable_values (when human-readable-values?
                                         (map second value-pairs)))))

(api/defendpoint POST 
  "Update the fields values and human-readable values for a `Field` whose semantic type is
  `category`/`city`/`state`/`country` or whose base type is `type/Boolean`. The human-readable values are optional."
  [id :as {{value-pairs :values} :body}]
  {id          ms/PositiveInt
   value-pairs [:sequential [:or [:tuple :any] [:tuple :any ms/NonBlankString]]]}
  (let [field (api/write-check Field id)]
    (api/check (field-values/field-should-have-field-values? field)
      [400 (str "You can only update the human readable values of a mapped values of a Field whose value of "
                "`has_field_values` is `list` or whose 'base_type' is 'type/Boolean'.")])
    (if-let [field-value-id (t2/select-one-pk FieldValues, :field_id id :type :full)]
      (update-field-values! field-value-id value-pairs)
      (create-field-values! field value-pairs)))
  {:status :success})

(api/defendpoint POST 
  "Manually trigger an update for the FieldValues for this Field. Only applies to Fields that are eligible for
   FieldValues."
  [id]
  {id ms/PositiveInt}
  (let [field (api/write-check (t2/select-one Field :id id))]
    ;; Override *current-user-permissions-set* so that permission checks pass during sync. If a user has DB detail perms
    ;; but no data perms, they should stll be able to trigger a sync of field values. This is fine because we don't
    ;; return any actual field values from this API. (#21764)
    (binding [api/*current-user-permissions-set* (atom #{"/"})]
      (field-values/create-or-update-full-field-values! field)))
  {:status :success})